refactor: utils organizations and add reference field in CSV sync

README.md

@@ -10,7 +10,7 @@ And with some specifics features:
 - implement helloasso custom fields mapping to paheko custom fields
 - avoid duplication
 
-Written in Rust.
+Written in ust.
 
 ## Getting started
 
@@ -18,7 +18,9 @@ Create a `.env` file from `.env.example` and fill in some secrets.
 
 Run the program `dotenv cargo run`
 
-## fonctionnements
+## Fonctionnement
+
+Arbre de décision au moment de l'import :
 
 - On va déjà récupérer la liste des utilisateurs en mode "summary" UserSummary` cad avec juste l'id, l'email, le prénom et le nom.
 - Ensuite on va récupérer la liste des transactions avec helloasso pour cette période comptable
@@ -44,3 +46,17 @@ ya une table `acc_transactions_users` qui permet de lier une transaction avec un
 
 le `order.id` et le `answer.id` que retourne l'API d'helloasso sont en fait les mêmes,
 
+
+## Import from CSV
+
+```
+xlsx2csv -n "Adhérents PAPIER" ./wip_matthieu_b.xlsx > ./to_import.csv
+
+```
+```
+cat ./tmp/adhesions_papier_nov2023.csv | cargo run -- --source csv
+```
+
+```
+cat ~/.mnt/etoiledebethleem/copyparty/adhesions/to_import.csv | head -n 25 | sed ':a;N;$!ba;s/\(Champ complémentaire [0-9]\)\n/\1 /g' | sed 's/Champ complémentaire \([0-9]\)/CC \1/g'
+```

TODO.md

@@ -12,6 +12,9 @@ like rossman said, you need to split up things
 - Lint, format code
 - Remove uneeded deps
 
+- find a way to export excel sheet from CSV
+    - `libreoffice --headless --convert-to 'csv' --outdir csv_exports ./ADH\ 02\ 01\ 2024.xlsx`
+
 # schedule
 
 ## 2023-12-23

src/main.rs

@@ -7,14 +7,17 @@ mod sync_helloasso;
 mod sync_csv;
 mod sync_paheko;
 
+#[cfg(test)]
+mod test_utils;
+
 use thiserror::Error;
 use anyhow::{Context, Result, anyhow};
-use chrono::prelude::{NaiveDate, Datelike};
 use strum::Display;
 use serde::{Serialize, Deserialize};
 use url::Url;
 use fully_pub::fully_pub;
 use argh::FromArgs;
+use utils::{parse_normalize_phone, normalize_str, parse_and_get_birthday_year};
 
 /// permanent config to store long-term config
 /// used to ingest env settings
@@ -126,31 +129,6 @@ async fn get_auth_client_from_cache(
     }
 }
 
-fn parse_normalize_phone(inp: String) -> Option<String> {
-    let parsed = match phonenumber::parse(
-        Some(phonenumber::country::Id::FR), inp
-    ) {
-        Ok(r) => r,
-        Err(_e) => {
-            return None;
-        }
-    };
-
-    Some(parsed.to_string())
-}
-
-
-fn normalize_str(subject: String) -> String {
-    subject.trim().replace("\n", ";").to_string()
-}
-
-/// remove year precision to comply with GDPR eu rules
-fn parse_and_get_birthday_year(raw_date: String) -> Option<u32> {
-    let d_res = NaiveDate::parse_from_str(raw_date.trim(), "%d/%m/%Y");
-    let d = d_res.ok()?;
-    d.year().try_into().ok()
-}
-
 fn get_proxy_from_url(proxy_url: &Option<String>) -> Result<Option<reqwest::Proxy>> {
     Ok(match proxy_url {
         Some(p) => Some(reqwest::Proxy::all(p)

src/sync_csv.rs

@@ -1,17 +1,35 @@
 use crate::paheko;
 use crate::{
     Config, UserCache,
-    parse_normalize_phone, normalize_str
 };
 
 use anyhow::Result;
-use crate::utils::parse_datetime_american;
+use crate::utils::{parse_datetime_american, normalize_str, parse_normalize_phone};
 use crate::sync_paheko::{GeneralizedAnswer, sync_paheko};
 use email_address::EmailAddress;
 use chrono::prelude::Datelike;
+use std::io::BufRead;
+use csv::ReaderBuilder;
 
 use std::io;
 
+fn process_csv_value(value: String) -> Option<String> {
+    let value = normalize_str(value);
+    if value.is_empty() {
+        return None
+    }
+    return Some(value)
+}
+
+fn process_price(value: String) -> f64 {
+    value
+        .trim()
+        .chars().filter(|c| c.is_numeric() || *c == '.')
+        .collect::<String>()
+        .parse().unwrap_or(0.0)
+}
+
+
 // read csv from stdin
 pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Config, user_cache: &mut UserCache) -> Result<()> {
     // raw row record directly from CSV
@@ -35,88 +53,82 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi
         #[serde(rename = "Don (€)")]
         donation_amount: String,
 
-        #[serde(rename = "Champ complémentaire 1 Prénom conjoint")]
+        #[serde(rename = "CC 1 Prénom conjoint")]
         linked_user_first_name: String,
 
-        #[serde(rename = "Champ complémentaire 2 ADRESSE")]
+        #[serde(rename = "CC 2 ADRESSE")]
         address: String,
-        #[serde(rename = "Champ complémentaire 3 CODE POSTAL")]
+        #[serde(rename = "CC 3 CODE POSTAL")]
         postal_code: String,
-        #[serde(rename = "Champ complémentaire 4 VILLE")]
+        #[serde(rename = "CC 4 VILLE")]
         city: String,
-        #[serde(rename = "Champ complémentaire 5 TÉLÉPHONE")]
+        #[serde(rename = "CC 5 TÉLÉPHONE")]
         phone: String,
 
-        #[serde(rename = "Champ complémentaire 7 PROFESSION")]
+        #[serde(rename = "CC 7 PROFESSION")]
         job: String,
-        #[serde(rename = "Champ complémentaire 8 CENTRE D'INTÉRÊTS / COMPÉTENCES")]
+        #[serde(rename = "CC 8 CENTRE D'INTÉRÊTS / COMPÉTENCES")]
         skills: String,
-        #[serde(rename = "Champ complémentaire 9 DATE DE NAISSANCE")]
+        #[serde(rename = "CC 9 DATE DE NAISSANCE")]
         birth_date: String,
 
-        #[serde(rename = "Référence")]
+        #[serde(rename = "REF BP/")]
         reference: String
     }
-    let mut rdr = csv::Reader::from_reader(io::stdin());
+    
+    let stdin = io::stdin();
+    let mut intermediate_inp = "".to_string();
+    for line_res in stdin.lock().lines() {
+        let line = line_res.unwrap();
+        if line.starts_with(",") {
+            continue;
+        }
+        if line.contains(&"\\FIN_DES_DONNES") {
+            break;
+        }
+        intermediate_inp.push_str(&line);
+        intermediate_inp.push_str("\n");
+    }
+
+    let mut rdr = ReaderBuilder::new()
+        .from_reader(intermediate_inp.as_bytes());
+
     let mut generalized_answers: Vec<GeneralizedAnswer> = vec![];
 
-    fn process_csv_value(value: String) -> Option<String> {
-        let value = normalize_str(value);
-        if value.is_empty() {
-            return None
-        }
-        return Some(value)
-    }
+    for parsed_record_res in rdr.deserialize() {
+        let parsed_record: AnswerRecord = parsed_record_res?;
+        println!("parsed_record: {:?}", parsed_record);
 
-    fn process_price(value: String) -> f64 {
-        value
-            .trim()
-            .chars().filter(|c| c.is_numeric() || *c == '.')
-            .collect::<String>()
-            .parse().unwrap_or(0.0)
-    }
-
-    for result in rdr.deserialize() {
-        let record: AnswerRecord = result?;
-        println!("{:?}", record);
-
-        let mut generalized_answer = GeneralizedAnswer {
-            first_name: Some(normalize_str(record.first_name)),
-            last_name: normalize_str(record.last_name),
-            email: process_csv_value(record.email).and_then(|s| EmailAddress::is_valid(&s).then(|| s)),
-            phone: process_csv_value(record.phone).and_then(|s| parse_normalize_phone(s)),
-            skills: process_csv_value(record.skills),
-            address: process_csv_value(record.address)
+        let generalized_answer = GeneralizedAnswer {
+            first_name: Some(normalize_str(parsed_record.first_name)),
+            last_name: normalize_str(parsed_record.last_name),
+            email: process_csv_value(parsed_record.email).and_then(|s| EmailAddress::is_valid(&s).then(|| s)),
+            phone: process_csv_value(parsed_record.phone).and_then(|s| parse_normalize_phone(s)),
+            skills: process_csv_value(parsed_record.skills),
+            address: process_csv_value(parsed_record.address)
                 .expect("Expected answer to have address"),
-            postal_code: process_csv_value(record.postal_code)
+            postal_code: process_csv_value(parsed_record.postal_code)
                 .expect("Expected answer to have postalcode"),
-            city: process_csv_value(record.city)
+            city: process_csv_value(parsed_record.city)
                 .expect("Expected answer answer to have city"),
             country: "fr".to_string(),
-            job: process_csv_value(record.job),
-            birth_year: process_csv_value(record.birth_date)
+            job: process_csv_value(parsed_record.job),
+            birth_year: process_csv_value(parsed_record.birth_date)
                 .and_then(|raw_date| parse_datetime_american(&raw_date))
                 .map(|d| d.year() as u32),
-            inception_time: process_csv_value(record.date)
+            inception_time: process_csv_value(parsed_record.date)
                 .map(|s| 
                     parse_datetime_american(&s).expect("Record must have a valid date")
                 )
                 .expect("Record must have a date"),
-            reference: format!("BP/{}", process_csv_value(record.reference).expect("Row must have reference")), // BP as Bulletin Papier
-            donation_amount: process_price(record.donation_amount),
-            subscription_amount: process_price(record.subscription_amount), // FIXME: get subscription from mode
-            membership_mode: serde_json::from_value(serde_json::Value::String(record.membership_mode.clone()))
+            reference: format!("BP/{}", process_csv_value(parsed_record.reference).expect("Row must have reference")), // BP as Bulletin Papier
+            donation_amount: process_price(parsed_record.donation_amount),
+            subscription_amount: process_price(parsed_record.subscription_amount), // FIXME: get subscription from mode
+            membership_mode: serde_json::from_value(serde_json::Value::String(parsed_record.membership_mode.clone()))
                 .expect("Expected a membership mode to be valid"),
-            linked_user_first_name: process_csv_value(record.linked_user_first_name)
+            linked_user_first_name: process_csv_value(parsed_record.linked_user_first_name)
         };
 
-        // apply custom user override
-        // this particular answer had duplicate phone and email from another answer
-        // if answer.id == 64756582 {
-        //     generalized_answer.email = None;
-        //     generalized_answer.phone = None;
-        // }
-
         generalized_answers.push(generalized_answer);
     }
     println!("Generated GeneralizedAnswers");

src/sync_helloasso.rs

@@ -2,8 +2,9 @@ use crate::helloasso;
 use crate::paheko;
 use crate::{
     Config, UserCache,
-    get_proxy_from_url, get_auth_client_from_cache, parse_and_get_birthday_year, parse_normalize_phone, normalize_str
+    get_proxy_from_url, get_auth_client_from_cache, 
 };
+use crate::utils::{parse_and_get_birthday_year, parse_normalize_phone, normalize_str};
 use crate::sync_paheko::{GeneralizedAnswer, sync_paheko};
 
 use anyhow::Result;

src/sync_paheko.rs

@@ -3,11 +3,10 @@ use crate::paheko::AccountingYear;
 use crate::{
     Config, UserCache,
 };
-use crate::utils;
+use crate::utils::{generate_id, normalize_first_name, normalize_last_name};
 
 use anyhow::{Context, Result};
 use chrono::prelude::{NaiveDate, DateTime, Utc};
-use crate::utils::generate_id;
 use fully_pub::fully_pub;
 use serde::{Serialize, Deserialize};
 
@@ -94,16 +93,24 @@ pub async fn sync_paheko(
     let mut pk_next_user_service_id = paheko_client.get_next_id("services_users")
         .await.context("Get paheko services_users next id")?;
 
-    for answer in answers {
+    for answer_inp in answers {
+        let mut answer = answer_inp;
+        answer.first_name = answer.first_name.map(normalize_first_name);
+        answer.last_name = normalize_last_name(answer.last_name);
+
         eprintln!("Processing answer:");
         eprintln!(" email: {:?}", answer.email);
+        eprintln!(" name: {} {}", &answer.last_name, answer.first_name.clone().unwrap_or("".to_string()));
 
         // list of users involved in this answer
         let mut pk_users_summaries: Vec<paheko::UserSummary> = vec![];
         let mut pk_user_service_registrations: Vec<paheko::UserServiceRegistration> = vec![];
 
         // check for existing user in paheko by email
-        let existing_user_opt = existing_users.iter().find(|user| user.email == answer.email).cloned();
+        // TODO: check user with fuzzing
+        let existing_user_opt = existing_users
+            .iter().find(|user| user.email == answer.email)
+            .cloned();
 
         // check for existing transactions
         if existing_transactions.iter().any(
@@ -222,7 +229,7 @@ pub async fn sync_paheko(
             accounting_year: get_accounting_year_for_time(&accounting_years, &answer.inception_time)
                     .expect("Cannot find an accounting year that match the date on paheko").id.clone(),
             // TODO: make the label template configurable
-            label: format!("Adhésion {:?} via {}", pk_membership.mode_name, via_name),
+            label: format!("{} {:?} via {}", pk_membership.service_name, pk_membership.mode_name, via_name),
             amount: pk_membership.payed_amount,
             reference: answer.reference,
             // TODO: make these field configurable
@@ -231,7 +238,7 @@ pub async fn sync_paheko(
             inception_time: answer.inception_time,
             kind: paheko::TransactionKind::Revenue,
             linked_users: pk_users_summaries.iter().map(|x| x.id.clone()).collect(),
-            // this depend on a patch to paheko API code to work
+            // the linked_services, depend on a patch to paheko API code to work (see https://forge.lefuturiste.fr/mbess/paheko-fork/commit/a4fdd816112f51db23a2b02ac160b0513a5b09c5)
             linked_services: pk_user_service_registrations.iter().map(|x| x.id.clone()).collect()
         };
         let _ = paheko_client.register_transaction(transaction)
@@ -240,6 +247,8 @@ pub async fn sync_paheko(
 
         stats.subscriptions_created += 1;
 
+        // TODO: handle donation amount
+
         pk_memberships.push(pk_membership);
     }
     eprintln!("{via_name} sync done.");

src/test_utils.rs

@@ -0,0 +1,21 @@
+use crate::utils::{normalize_str, normalize_first_name};
+
+#[test]
+fn test_normalize_str() {
+    let out = normalize_str("    hello world   ".to_string());
+
+    assert_eq!(out, "hello world");
+}
+
+#[test]
+fn test_normalize_first_name() {
+    let out = normalize_first_name("JEAN-PIERRE".to_string());
+    assert_eq!(out, "Jean-Pierre");
+
+    let out = normalize_first_name("JEAN PIERRE".to_string());
+    assert_eq!(out, "Jean-Pierre");
+
+    let out = normalize_first_name("jeffrey".to_string());
+    assert_eq!(out, "Jeffrey");
+}
+

src/utils.rs

@@ -1,7 +1,7 @@
 use serde::{Serialize, Deserialize, Deserializer};
 use std::fmt;
 use rand::{thread_rng, Rng};
-use chrono::prelude::{DateTime, Utc, NaiveDate};
+use chrono::prelude::{DateTime, Utc, NaiveDate, Datelike};
 
 /// ID
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq, Hash)]
@@ -59,3 +59,52 @@ pub fn parse_datetime_american(inp: &str) -> Option<DateTime<Utc>> {
         Utc
     ))
 }
+
+pub fn parse_normalize_phone(inp: String) -> Option<String> {
+    let parsed = match phonenumber::parse(
+        Some(phonenumber::country::Id::FR), inp
+    ) {
+        Ok(r) => r,
+        Err(_e) => {
+            return None;
+        }
+    };
+
+    Some(parsed.to_string())
+}
+
+
+pub fn normalize_str(subject: String) -> String {
+    subject.trim().replace("\n", ";").to_string()
+}
+
+/// remove year precision to comply with GDPR eu rules
+pub fn parse_and_get_birthday_year(raw_date: String) -> Option<u32> {
+    let d_res = NaiveDate::parse_from_str(raw_date.trim(), "%d/%m/%Y");
+    let d = d_res.ok()?;
+    d.year().try_into().ok()
+}
+
+pub fn capitalize(s: &str) -> String {
+    let mut c = s.chars();
+    match c.next() {
+        None => String::new(),
+        Some(f) => f.to_uppercase().collect::<String>() + c.as_str(),
+    }
+}
+
+pub fn normalize_first_name(subject: String) -> String {
+    subject
+        .to_lowercase()
+        .replace(" ", "-")
+        .split("-")
+        .map(|x| capitalize(x))
+        .collect::<Vec<String>>()
+        .join("-")
+}
+
+pub fn normalize_last_name(subject: String) -> String {
+    subject
+        .to_uppercase()
+}
+