From bbea78cea302891c983c854547c9480a977764d3 Mon Sep 17 00:00:00 2001 From: Matthieu Bessat Date: Fri, 19 Jan 2024 12:54:24 +0100 Subject: [PATCH] refactor: utils organizations and add reference field in CSV sync --- README.md | 20 ++++++- TODO.md | 3 ++ src/main.rs | 30 ++--------- src/sync_csv.rs | 122 +++++++++++++++++++++++------------------- src/sync_helloasso.rs | 3 +- src/sync_paheko.rs | 21 +++++--- src/test_utils.rs | 21 ++++++++ src/utils.rs | 51 +++++++++++++++++- 8 files changed, 180 insertions(+), 91 deletions(-) create mode 100644 src/test_utils.rs diff --git a/README.md b/README.md index 5b92585..d1c41a4 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ And with some specifics features: - implement helloasso custom fields mapping to paheko custom fields - avoid duplication -Written in Rust. +Written in ust. ## Getting started @@ -18,7 +18,9 @@ Create a `.env` file from `.env.example` and fill in some secrets. Run the program `dotenv cargo run` -## fonctionnements +## Fonctionnement + +Arbre de décision au moment de l'import : - On va déjà récupérer la liste des utilisateurs en mode "summary" UserSummary` cad avec juste l'id, l'email, le prénom et le nom. - Ensuite on va récupérer la liste des transactions avec helloasso pour cette période comptable @@ -44,3 +46,17 @@ ya une table `acc_transactions_users` qui permet de lier une transaction avec un le `order.id` et le `answer.id` que retourne l'API d'helloasso sont en fait les mêmes, + +## Import from CSV + +``` +xlsx2csv -n "Adhérents PAPIER" ./wip_matthieu_b.xlsx > ./to_import.csv + +``` +``` +cat ./tmp/adhesions_papier_nov2023.csv | cargo run -- --source csv +``` + +``` +cat ~/.mnt/etoiledebethleem/copyparty/adhesions/to_import.csv | head -n 25 | sed ':a;N;$!ba;s/\(Champ complémentaire [0-9]\)\n/\1 /g' | sed 's/Champ complémentaire \([0-9]\)/CC \1/g' +``` diff --git a/TODO.md b/TODO.md index eda1acb..24b1b80 100644 --- a/TODO.md +++ b/TODO.md @@ -12,6 +12,9 @@ like rossman said, you need to split up things - Lint, format code - Remove uneeded deps +- find a way to export excel sheet from CSV + - `libreoffice --headless --convert-to 'csv' --outdir csv_exports ./ADH\ 02\ 01\ 2024.xlsx` + # schedule ## 2023-12-23 diff --git a/src/main.rs b/src/main.rs index 98d0407..aaa3ca2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,14 +7,17 @@ mod sync_helloasso; mod sync_csv; mod sync_paheko; +#[cfg(test)] +mod test_utils; + use thiserror::Error; use anyhow::{Context, Result, anyhow}; -use chrono::prelude::{NaiveDate, Datelike}; use strum::Display; use serde::{Serialize, Deserialize}; use url::Url; use fully_pub::fully_pub; use argh::FromArgs; +use utils::{parse_normalize_phone, normalize_str, parse_and_get_birthday_year}; /// permanent config to store long-term config /// used to ingest env settings @@ -126,31 +129,6 @@ async fn get_auth_client_from_cache( } } -fn parse_normalize_phone(inp: String) -> Option { - let parsed = match phonenumber::parse( - Some(phonenumber::country::Id::FR), inp - ) { - Ok(r) => r, - Err(_e) => { - return None; - } - }; - - Some(parsed.to_string()) -} - - -fn normalize_str(subject: String) -> String { - subject.trim().replace("\n", ";").to_string() -} - -/// remove year precision to comply with GDPR eu rules -fn parse_and_get_birthday_year(raw_date: String) -> Option { - let d_res = NaiveDate::parse_from_str(raw_date.trim(), "%d/%m/%Y"); - let d = d_res.ok()?; - d.year().try_into().ok() -} - fn get_proxy_from_url(proxy_url: &Option) -> Result> { Ok(match proxy_url { Some(p) => Some(reqwest::Proxy::all(p) diff --git a/src/sync_csv.rs b/src/sync_csv.rs index 290ab1e..fa440e8 100644 --- a/src/sync_csv.rs +++ b/src/sync_csv.rs @@ -1,17 +1,35 @@ use crate::paheko; use crate::{ Config, UserCache, - parse_normalize_phone, normalize_str }; use anyhow::Result; -use crate::utils::parse_datetime_american; +use crate::utils::{parse_datetime_american, normalize_str, parse_normalize_phone}; use crate::sync_paheko::{GeneralizedAnswer, sync_paheko}; use email_address::EmailAddress; use chrono::prelude::Datelike; +use std::io::BufRead; +use csv::ReaderBuilder; use std::io; +fn process_csv_value(value: String) -> Option { + let value = normalize_str(value); + if value.is_empty() { + return None + } + return Some(value) +} + +fn process_price(value: String) -> f64 { + value + .trim() + .chars().filter(|c| c.is_numeric() || *c == '.') + .collect::() + .parse().unwrap_or(0.0) +} + + // read csv from stdin pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Config, user_cache: &mut UserCache) -> Result<()> { // raw row record directly from CSV @@ -35,88 +53,82 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi #[serde(rename = "Don (€)")] donation_amount: String, - #[serde(rename = "Champ complémentaire 1 Prénom conjoint")] + #[serde(rename = "CC 1 Prénom conjoint")] linked_user_first_name: String, - #[serde(rename = "Champ complémentaire 2 ADRESSE")] + #[serde(rename = "CC 2 ADRESSE")] address: String, - #[serde(rename = "Champ complémentaire 3 CODE POSTAL")] + #[serde(rename = "CC 3 CODE POSTAL")] postal_code: String, - #[serde(rename = "Champ complémentaire 4 VILLE")] + #[serde(rename = "CC 4 VILLE")] city: String, - #[serde(rename = "Champ complémentaire 5 TÉLÉPHONE")] + #[serde(rename = "CC 5 TÉLÉPHONE")] phone: String, - #[serde(rename = "Champ complémentaire 7 PROFESSION")] + #[serde(rename = "CC 7 PROFESSION")] job: String, - #[serde(rename = "Champ complémentaire 8 CENTRE D'INTÉRÊTS / COMPÉTENCES")] + #[serde(rename = "CC 8 CENTRE D'INTÉRÊTS / COMPÉTENCES")] skills: String, - #[serde(rename = "Champ complémentaire 9 DATE DE NAISSANCE")] + #[serde(rename = "CC 9 DATE DE NAISSANCE")] birth_date: String, - #[serde(rename = "Référence")] + #[serde(rename = "REF BP/")] reference: String } - let mut rdr = csv::Reader::from_reader(io::stdin()); + + let stdin = io::stdin(); + let mut intermediate_inp = "".to_string(); + for line_res in stdin.lock().lines() { + let line = line_res.unwrap(); + if line.starts_with(",") { + continue; + } + if line.contains(&"\\FIN_DES_DONNES") { + break; + } + intermediate_inp.push_str(&line); + intermediate_inp.push_str("\n"); + } + + let mut rdr = ReaderBuilder::new() + .from_reader(intermediate_inp.as_bytes()); + let mut generalized_answers: Vec = vec![]; - fn process_csv_value(value: String) -> Option { - let value = normalize_str(value); - if value.is_empty() { - return None - } - return Some(value) - } + for parsed_record_res in rdr.deserialize() { + let parsed_record: AnswerRecord = parsed_record_res?; + println!("parsed_record: {:?}", parsed_record); - fn process_price(value: String) -> f64 { - value - .trim() - .chars().filter(|c| c.is_numeric() || *c == '.') - .collect::() - .parse().unwrap_or(0.0) - } - - for result in rdr.deserialize() { - let record: AnswerRecord = result?; - println!("{:?}", record); - - let mut generalized_answer = GeneralizedAnswer { - first_name: Some(normalize_str(record.first_name)), - last_name: normalize_str(record.last_name), - email: process_csv_value(record.email).and_then(|s| EmailAddress::is_valid(&s).then(|| s)), - phone: process_csv_value(record.phone).and_then(|s| parse_normalize_phone(s)), - skills: process_csv_value(record.skills), - address: process_csv_value(record.address) + let generalized_answer = GeneralizedAnswer { + first_name: Some(normalize_str(parsed_record.first_name)), + last_name: normalize_str(parsed_record.last_name), + email: process_csv_value(parsed_record.email).and_then(|s| EmailAddress::is_valid(&s).then(|| s)), + phone: process_csv_value(parsed_record.phone).and_then(|s| parse_normalize_phone(s)), + skills: process_csv_value(parsed_record.skills), + address: process_csv_value(parsed_record.address) .expect("Expected answer to have address"), - postal_code: process_csv_value(record.postal_code) + postal_code: process_csv_value(parsed_record.postal_code) .expect("Expected answer to have postalcode"), - city: process_csv_value(record.city) + city: process_csv_value(parsed_record.city) .expect("Expected answer answer to have city"), country: "fr".to_string(), - job: process_csv_value(record.job), - birth_year: process_csv_value(record.birth_date) + job: process_csv_value(parsed_record.job), + birth_year: process_csv_value(parsed_record.birth_date) .and_then(|raw_date| parse_datetime_american(&raw_date)) .map(|d| d.year() as u32), - inception_time: process_csv_value(record.date) + inception_time: process_csv_value(parsed_record.date) .map(|s| parse_datetime_american(&s).expect("Record must have a valid date") ) .expect("Record must have a date"), - reference: format!("BP/{}", process_csv_value(record.reference).expect("Row must have reference")), // BP as Bulletin Papier - donation_amount: process_price(record.donation_amount), - subscription_amount: process_price(record.subscription_amount), // FIXME: get subscription from mode - membership_mode: serde_json::from_value(serde_json::Value::String(record.membership_mode.clone())) + reference: format!("BP/{}", process_csv_value(parsed_record.reference).expect("Row must have reference")), // BP as Bulletin Papier + donation_amount: process_price(parsed_record.donation_amount), + subscription_amount: process_price(parsed_record.subscription_amount), // FIXME: get subscription from mode + membership_mode: serde_json::from_value(serde_json::Value::String(parsed_record.membership_mode.clone())) .expect("Expected a membership mode to be valid"), - linked_user_first_name: process_csv_value(record.linked_user_first_name) + linked_user_first_name: process_csv_value(parsed_record.linked_user_first_name) }; - // apply custom user override - // this particular answer had duplicate phone and email from another answer - // if answer.id == 64756582 { - // generalized_answer.email = None; - // generalized_answer.phone = None; - // } - generalized_answers.push(generalized_answer); } println!("Generated GeneralizedAnswers"); diff --git a/src/sync_helloasso.rs b/src/sync_helloasso.rs index ad9451d..aaa7ec1 100644 --- a/src/sync_helloasso.rs +++ b/src/sync_helloasso.rs @@ -2,8 +2,9 @@ use crate::helloasso; use crate::paheko; use crate::{ Config, UserCache, - get_proxy_from_url, get_auth_client_from_cache, parse_and_get_birthday_year, parse_normalize_phone, normalize_str + get_proxy_from_url, get_auth_client_from_cache, }; +use crate::utils::{parse_and_get_birthday_year, parse_normalize_phone, normalize_str}; use crate::sync_paheko::{GeneralizedAnswer, sync_paheko}; use anyhow::Result; diff --git a/src/sync_paheko.rs b/src/sync_paheko.rs index 83bfb53..33945cc 100644 --- a/src/sync_paheko.rs +++ b/src/sync_paheko.rs @@ -3,11 +3,10 @@ use crate::paheko::AccountingYear; use crate::{ Config, UserCache, }; -use crate::utils; +use crate::utils::{generate_id, normalize_first_name, normalize_last_name}; use anyhow::{Context, Result}; use chrono::prelude::{NaiveDate, DateTime, Utc}; -use crate::utils::generate_id; use fully_pub::fully_pub; use serde::{Serialize, Deserialize}; @@ -94,16 +93,24 @@ pub async fn sync_paheko( let mut pk_next_user_service_id = paheko_client.get_next_id("services_users") .await.context("Get paheko services_users next id")?; - for answer in answers { + for answer_inp in answers { + let mut answer = answer_inp; + answer.first_name = answer.first_name.map(normalize_first_name); + answer.last_name = normalize_last_name(answer.last_name); + eprintln!("Processing answer:"); eprintln!(" email: {:?}", answer.email); + eprintln!(" name: {} {}", &answer.last_name, answer.first_name.clone().unwrap_or("".to_string())); // list of users involved in this answer let mut pk_users_summaries: Vec = vec![]; let mut pk_user_service_registrations: Vec = vec![]; // check for existing user in paheko by email - let existing_user_opt = existing_users.iter().find(|user| user.email == answer.email).cloned(); + // TODO: check user with fuzzing + let existing_user_opt = existing_users + .iter().find(|user| user.email == answer.email) + .cloned(); // check for existing transactions if existing_transactions.iter().any( @@ -222,7 +229,7 @@ pub async fn sync_paheko( accounting_year: get_accounting_year_for_time(&accounting_years, &answer.inception_time) .expect("Cannot find an accounting year that match the date on paheko").id.clone(), // TODO: make the label template configurable - label: format!("Adhésion {:?} via {}", pk_membership.mode_name, via_name), + label: format!("{} {:?} via {}", pk_membership.service_name, pk_membership.mode_name, via_name), amount: pk_membership.payed_amount, reference: answer.reference, // TODO: make these field configurable @@ -231,7 +238,7 @@ pub async fn sync_paheko( inception_time: answer.inception_time, kind: paheko::TransactionKind::Revenue, linked_users: pk_users_summaries.iter().map(|x| x.id.clone()).collect(), - // this depend on a patch to paheko API code to work + // the linked_services, depend on a patch to paheko API code to work (see https://forge.lefuturiste.fr/mbess/paheko-fork/commit/a4fdd816112f51db23a2b02ac160b0513a5b09c5) linked_services: pk_user_service_registrations.iter().map(|x| x.id.clone()).collect() }; let _ = paheko_client.register_transaction(transaction) @@ -240,6 +247,8 @@ pub async fn sync_paheko( stats.subscriptions_created += 1; + // TODO: handle donation amount + pk_memberships.push(pk_membership); } eprintln!("{via_name} sync done."); diff --git a/src/test_utils.rs b/src/test_utils.rs new file mode 100644 index 0000000..5121b04 --- /dev/null +++ b/src/test_utils.rs @@ -0,0 +1,21 @@ +use crate::utils::{normalize_str, normalize_first_name}; + +#[test] +fn test_normalize_str() { + let out = normalize_str(" hello world ".to_string()); + + assert_eq!(out, "hello world"); +} + +#[test] +fn test_normalize_first_name() { + let out = normalize_first_name("JEAN-PIERRE".to_string()); + assert_eq!(out, "Jean-Pierre"); + + let out = normalize_first_name("JEAN PIERRE".to_string()); + assert_eq!(out, "Jean-Pierre"); + + let out = normalize_first_name("jeffrey".to_string()); + assert_eq!(out, "Jeffrey"); +} + diff --git a/src/utils.rs b/src/utils.rs index 4354e7e..006b478 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,7 +1,7 @@ use serde::{Serialize, Deserialize, Deserializer}; use std::fmt; use rand::{thread_rng, Rng}; -use chrono::prelude::{DateTime, Utc, NaiveDate}; +use chrono::prelude::{DateTime, Utc, NaiveDate, Datelike}; /// ID #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq, Hash)] @@ -59,3 +59,52 @@ pub fn parse_datetime_american(inp: &str) -> Option> { Utc )) } + +pub fn parse_normalize_phone(inp: String) -> Option { + let parsed = match phonenumber::parse( + Some(phonenumber::country::Id::FR), inp + ) { + Ok(r) => r, + Err(_e) => { + return None; + } + }; + + Some(parsed.to_string()) +} + + +pub fn normalize_str(subject: String) -> String { + subject.trim().replace("\n", ";").to_string() +} + +/// remove year precision to comply with GDPR eu rules +pub fn parse_and_get_birthday_year(raw_date: String) -> Option { + let d_res = NaiveDate::parse_from_str(raw_date.trim(), "%d/%m/%Y"); + let d = d_res.ok()?; + d.year().try_into().ok() +} + +pub fn capitalize(s: &str) -> String { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().collect::() + c.as_str(), + } +} + +pub fn normalize_first_name(subject: String) -> String { + subject + .to_lowercase() + .replace(" ", "-") + .split("-") + .map(|x| capitalize(x)) + .collect::>() + .join("-") +} + +pub fn normalize_last_name(subject: String) -> String { + subject + .to_uppercase() +} +