diff --git a/TODO.md b/TODO.md index 24b1b80..89ce09e 100644 --- a/TODO.md +++ b/TODO.md @@ -53,3 +53,8 @@ TODO: query all subscriptions of user byu service label curl -u $PAHEKO_CLIENT_ID:$PAHEKO_CLIENT_SECRET http://localhost:8082/api/sql -d "SELECT su.id_user,su.date FROM services_users AS su JOIN services AS s ON su.id_service = s.id WHERE s.label = 'Cotisation 2023-2024';" + +## 2024-01-20 + +- handle additional donation amount + diff --git a/src/helloasso.rs b/src/helloasso.rs index 67b8623..5fea636 100644 --- a/src/helloasso.rs +++ b/src/helloasso.rs @@ -310,7 +310,10 @@ struct Payment { id: u64, #[serde(rename = "type")] - extra: Option + extra: Option, + + share_amount: u32, + amount: u32 } #[derive(Debug, Serialize, Deserialize)] diff --git a/src/main.rs b/src/main.rs index aaa3ca2..3fe866b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,6 @@ use serde::{Serialize, Deserialize}; use url::Url; use fully_pub::fully_pub; use argh::FromArgs; -use utils::{parse_normalize_phone, normalize_str, parse_and_get_birthday_year}; /// permanent config to store long-term config /// used to ingest env settings @@ -38,7 +37,7 @@ struct Config { paheko_client_secret: String, paheko_target_activity_name: String, - // paheko_accounting_year_id: u64, + paheko_accounting_years_ids: Vec, } // start user cache management @@ -137,10 +136,7 @@ fn get_proxy_from_url(proxy_url: &Option) -> Result Result<()> { - dotenvy::dotenv()?; - - let config: Config = envy::from_env().context("Failed to load env vars")?; +async fn launch_adapter(source: SourceType, config: &Config) -> Result<()> { let mut user_cache = load_user_cache().context("Failed to load user cache")?; @@ -173,7 +169,11 @@ async fn launch_adapter(source: SourceType) -> Result<()> { struct App { /// the source of sync (CSV or helloasso) #[argh(option, short = 'm')] - source: String, + source: Option, + + /// output debug info + #[argh(switch, short = 'i')] + info: bool } enum SourceType { @@ -184,7 +184,15 @@ enum SourceType { #[tokio::main] async fn main() { let app: App = argh::from_env(); - let source = match app.source.as_ref() { + dotenvy::dotenv().expect("Could not load dot env file"); + let config: Config = envy::from_env().expect("Failed to load env vars"); + + if app.info { + dbg!(config); + return; + } + + let source = match app.source.unwrap().as_ref() { "helloasso" => SourceType::Helloasso, "csv" => SourceType::Csv, _ => { @@ -192,7 +200,7 @@ async fn main() { return; } }; - let res = launch_adapter(source).await; + let res = launch_adapter(source, &config).await; match res { Err(err) => { eprintln!("Program failed, details bellow"); diff --git a/src/paheko.rs b/src/paheko.rs index e5e2e26..aa7901f 100644 --- a/src/paheko.rs +++ b/src/paheko.rs @@ -8,7 +8,7 @@ use chrono::prelude::{DateTime, Utc}; use chrono::NaiveDate; use thiserror::Error; use crate::sync_paheko::GeneralizedAnswer; -use crate::utils::deserialize_date; +use crate::utils::{deserialize_date, deserialize_json_list, complete_date}; #[derive(Debug, Serialize, Deserialize, Clone)] @@ -250,13 +250,6 @@ struct SqlQueryOutput { results: serde_json::Value } -#[derive(Debug, Deserialize)] -#[fully_pub] -struct TransactionSummary { - id: u64, - reference: String -} - #[derive(Debug)] #[fully_pub] struct UserServiceRegistration { @@ -320,16 +313,53 @@ impl AuthentifiedClient { Ok(ids.get(0).map(|x| x.id).unwrap_or(1)+1) } - pub async fn get_transactions(&self, id_year: u32) - -> Result> + pub async fn get_transactions(&self, id_years: &Vec) + -> Result> { + #[derive(Debug, Deserialize)] + struct Row { + id: u64, + label: String, + reference: String, + #[serde(deserialize_with = "deserialize_json_list")] + accounts_codes: Vec, + year_id: u64, + #[serde(deserialize_with = "deserialize_date")] + inception_date: NaiveDate + } + + let id_years_joined = id_years.iter() + .map(|x| x.to_string()) + .collect::>() + .join(","); let query: String = format!(r#" - SELECT id,reference FROM acc_transactions WHERE id_year={} AND reference LIKE 'HA/%'; - "#, id_year).to_string(); + SELECT act.id, act.date AS inception_date, act.id_year AS year_id, act.label, act.reference, acc.code, JSON_GROUP_ARRAY(acc.code) AS accounts_codes + FROM acc_transactions AS act + INNER JOIN acc_transactions_lines AS actl ON actl.id_transaction = act.id + INNER JOIN acc_accounts AS acc ON acc.id = actl.id_account + WHERE act.id_year IN ({}) + GROUP BY act.id; + "#, id_years_joined).to_string(); let val = self.sql_query(query).await.context("Fetching transactions")?; - Ok(serde_json::from_value(val.results)?) + let raw_vals: Vec = serde_json::from_value(val.results) + .context("Cannot deserialize SQL transactions rows")?; + + // we will assume that the first acc code is always the credit, and second the debit + + Ok(raw_vals.iter().map(|x| SimpleTransaction { + label: x.label.clone(), + reference: x.reference.clone(), + credit_account_code: x.accounts_codes.get(0).unwrap().to_string(), + debit_account_code: x.accounts_codes.get(1).unwrap().to_string(), + inception_time: complete_date(x.inception_date), + accounting_year: Id(x.year_id), + amount: 0.0, + kind: TransactionKind::Expense, + linked_subscriptions: vec![], + linked_users: vec![], + }).collect()) } pub async fn get_accounting_years(&self) diff --git a/src/sync_csv.rs b/src/sync_csv.rs index fa440e8..e4f6aba 100644 --- a/src/sync_csv.rs +++ b/src/sync_csv.rs @@ -13,6 +13,8 @@ use csv::ReaderBuilder; use std::io; +const CAISSE_ACCOUNT_CODE: &str = "530"; // 530 - Caisse + fn process_csv_value(value: String) -> Option { let value = normalize_str(value); if value.is_empty() { @@ -80,6 +82,7 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi let mut intermediate_inp = "".to_string(); for line_res in stdin.lock().lines() { let line = line_res.unwrap(); + eprintln!("{:?}",&line); if line.starts_with(",") { continue; } @@ -95,9 +98,11 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi let mut generalized_answers: Vec = vec![]; + eprintln!("Reading from stdin"); + for parsed_record_res in rdr.deserialize() { let parsed_record: AnswerRecord = parsed_record_res?; - println!("parsed_record: {:?}", parsed_record); + eprintln!("Parsed_record: {:?}", parsed_record); let generalized_answer = GeneralizedAnswer { first_name: Some(normalize_str(parsed_record.first_name)), @@ -131,13 +136,13 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi generalized_answers.push(generalized_answer); } - println!("Generated GeneralizedAnswers"); + eprintln!("Generated GeneralizedAnswers"); sync_paheko( paheko_client, config, user_cache, generalized_answers, - "530", + CAISSE_ACCOUNT_CODE, "Papier" ).await?; eprintln!("CSV sync done."); diff --git a/src/sync_helloasso.rs b/src/sync_helloasso.rs index dd7e595..ca4b4fe 100644 --- a/src/sync_helloasso.rs +++ b/src/sync_helloasso.rs @@ -89,12 +89,14 @@ pub async fn sync_helloasso(paheko_client: &paheko::AuthentifiedClient, config: let email = choose_email(&answer); // skip answers that were imported later and are stranger from helloasso - if let Some(payment) = answer.payments.iter().nth(0) { - if payment.extra == Some("Offline".to_string()) { - continue; - } + let payment = answer.payments.iter().nth(0).expect("Expected payment to exists"); + if payment.extra == Some("Offline".to_string()) { + continue; } + let subscription_amount = f64::from(payment.share_amount)/100.0; + let donation_amount = f64::from(payment.amount - payment.share_amount)/100.0; + let mut generalized_answer = GeneralizedAnswer { first_name: Some(normalize_str(answer.user.first_name.clone())), last_name: normalize_str(answer.user.last_name.clone()), @@ -117,8 +119,10 @@ pub async fn sync_helloasso(paheko_client: &paheko::AuthentifiedClient, config: birth_year: read_custom_field(&answer, HelloassoCustomFieldType::Birthday).and_then(parse_and_get_birthday_year), inception_time: answer.order.inception_time, reference: format!("HA/{}", answer.id), - donation_amount: 0.0, - subscription_amount: f64::from(answer.amount)/100.0, + // TODO: handle donation from helloasso, compare initial_amount and amount in payment + // or shareAmount + donation_amount, + subscription_amount, membership_mode: serde_json::from_value(serde_json::Value::String(answer.mode.clone())) .expect("Expected a membership mode to be valid"), linked_user_first_name: read_custom_field(&answer, HelloassoCustomFieldType::LinkedUserFirstName) diff --git a/src/sync_paheko.rs b/src/sync_paheko.rs index 539be41..33c3f74 100644 --- a/src/sync_paheko.rs +++ b/src/sync_paheko.rs @@ -1,5 +1,5 @@ use crate::paheko; -use crate::paheko::AccountingYear; +use crate::paheko::{AccountingYear, SimpleTransaction}; use crate::{ Config, UserCache, }; @@ -10,6 +10,9 @@ use chrono::prelude::{NaiveDate, DateTime, Utc}; use fully_pub::fully_pub; use serde::{Serialize, Deserialize}; +const DONATION_ACCOUNT_CODE: &str = "754"; +const SUBSCRIPTION_ACCOUNT_CODE: &str = "756"; + #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] #[fully_pub] enum MembershipMode { @@ -67,10 +70,11 @@ pub async fn sync_paheko( struct Stats { subscriptions_created: u32, + transaction_created: u32, users_created: u32 } - let mut stats = Stats { subscriptions_created: 0, users_created: 0 }; + let mut stats = Stats { transaction_created: 0, subscriptions_created: 0, users_created: 0 }; let mut pk_memberships: Vec = vec![]; @@ -80,7 +84,8 @@ pub async fn sync_paheko( let mut existing_users = paheko_client.get_users().await.context("Get users")?; // 2. get summary of transactions for that year // TODO: also get for the year n-1 - let existing_transactions = paheko_client.get_transactions(1).await.context("Get transactions")?; + let existing_transactions = paheko_client.get_transactions(&config.paheko_accounting_years_ids) + .await.context("Get transactions")?; // 3. get summary of services_users for that year let existing_subscriptions = paheko_client.get_service_subscriptions(&config.paheko_target_activity_name) .await.context("Get existing paheko subscriptions to the target activity")?; @@ -106,24 +111,21 @@ pub async fn sync_paheko( let mut pk_users_summaries: Vec = vec![]; let mut pk_user_service_registrations: Vec = vec![]; + + let existing_matching_transactions: Vec<&SimpleTransaction> = existing_transactions + .iter() + .filter(|t| t.reference == answer.reference) + .collect(); + + // dbg!(&existing_subscriptions); // check for existing user in paheko by email // TODO: check user with fuzzing first name and last name let existing_user_opt = existing_users .iter().find(|user| user.first_name == answer.first_name && user.last_name == answer.last_name) .cloned(); - - // check for existing transactions - if existing_transactions.iter().any( - |summary| summary.reference == answer.reference - ) { - eprintln!(" skipped: existing transaction found"); - continue; - } - - // dbg!(&existing_subscriptions); let pk_user_summary = match existing_user_opt.clone() { Some(user) => { - eprintln!(" Found existing paheko user by name."); + eprintln!(" Found existing paheko user by matching name."); user }, None => { @@ -139,20 +141,8 @@ pub async fn sync_paheko( } }; pk_users_summaries.push(pk_user_summary.clone()); - - // check if the user is already subscribed to the target activity - if - existing_user_opt.is_some() && - existing_subscriptions.iter() - .find(|membership| membership.users_ids - .iter().find(|i| **i == pk_user_summary.id).is_some() - ) - .is_some() - { - eprintln!(" skipped: user is already subscribed to this activity"); - continue; - } + let mut pk_membership = paheko::Membership { id: generate_id(), service_name: config.paheko_target_activity_name.clone(), @@ -168,96 +158,143 @@ pub async fn sync_paheko( payed_amount: answer.subscription_amount, users_ids: vec![pk_user_summary.id.clone()] }; + // check if the user is already subscribed to the target activity + if + existing_user_opt.is_some() && + existing_subscriptions.iter() + .find(|membership| membership.users_ids + .iter().find(|i| **i == pk_user_summary.id).is_some() + ) + .is_some() { + eprintln!(" User is already subscribed to this activity"); + } else { + // add activity for first member + // TODO: check if activity already exists + let user_registration = paheko_client.register_user_to_service( + pk_users_summaries.get(0).unwrap(), + &pk_membership, + pk_next_user_service_id + ).await.context("Expected to register user activity to paheko")?; + pk_user_service_registrations.push(user_registration); + pk_next_user_service_id += 1; + stats.subscriptions_created += 1; + eprintln!(" Created paheko activity registration"); - // add activity for first member - // TODO: check if activity already exists - let user_registration = paheko_client.register_user_to_service( - pk_users_summaries.get(0).unwrap(), - &pk_membership, - pk_next_user_service_id - ).await.context("Expected to register user activity to paheko")?; - pk_user_service_registrations.push(user_registration); - pk_next_user_service_id += 1; - eprintln!(" Created paheko activity registration"); - - // then create optional linked user - if answer.membership_mode == MembershipMode::Couple { - let mut second_answer = answer.clone(); - second_answer.email = None; - second_answer.phone = None; - second_answer.skills = None; - second_answer.job = None; - second_answer.birth_year = None; - - // add first_name - match answer.linked_user_first_name { - Some(name) => { - second_answer.first_name = Some(name); - }, - None => { - second_answer.first_name = None; - eprintln!("Warn: Got a user with Couple mode but no additional name given!") - } - } - - if existing_user_opt.is_none() { - // only create the linked user in paheko, if the first user was also created - let second_pk_user_summary = paheko_client.create_user(&second_answer, pk_next_user_id) - .await.context("Expected to create second paheko user")?; - eprintln!(" Created conjoint paheko user"); - pk_users_summaries.push(second_pk_user_summary.clone()); - pk_next_user_id += 1; - - // create activity of second user - let user_registration = paheko_client.register_user_to_service( - pk_users_summaries.get(1).unwrap(), // pass user, for the id - &pk_membership, - pk_next_user_service_id - ).await.context("Registering service to second paheko server")?; - pk_user_service_registrations.push(user_registration); - pk_next_user_service_id += 1; - eprintln!(" Created paheko activity registration for conjoint user"); - - pk_membership.users_ids.push(second_pk_user_summary.id) - } + // then create optional linked user // FIXME: reuse a previous user // TODO: get existing linked user from previous year + if answer.membership_mode == MembershipMode::Couple { + let mut second_answer = answer.clone(); + second_answer.email = None; + second_answer.phone = None; + second_answer.skills = None; + second_answer.job = None; + second_answer.birth_year = None; + // add first_name + match answer.linked_user_first_name { + Some(name) => { + second_answer.first_name = Some(name); + }, + None => { + second_answer.first_name = None; + eprintln!("Warn: Got a user with Couple mode but no additional name given!") + } + } + + if existing_user_opt.is_none() { + // only create the linked user in paheko, if the first user was also created + let second_pk_user_summary = paheko_client.create_user(&second_answer, pk_next_user_id) + .await.context("Expected to create second paheko user")?; + eprintln!(" Created conjoint paheko user"); + pk_users_summaries.push(second_pk_user_summary.clone()); + pk_next_user_id += 1; + + // create activity of second user + let user_registration = paheko_client.register_user_to_service( + pk_users_summaries.get(1).unwrap(), // pass user, for the id + &pk_membership, + pk_next_user_service_id + ).await.context("Registering service to second paheko server")?; + pk_user_service_registrations.push(user_registration); + pk_next_user_service_id += 1; + stats.subscriptions_created += 1; + eprintln!(" Created paheko activity registration for conjoint user"); + + pk_membership.users_ids.push(second_pk_user_summary.id) + } + } } - // add transaction - let transaction = paheko::SimpleTransaction { - accounting_year: match get_accounting_year_for_time(&accounting_years, &answer.inception_time) { - None => { - eprintln!("Cannot find an accounting year on paheko that include the inception date {:?} given", &answer.inception_time); - panic!(); - }, - Some(s) => s - }.id.clone(), - // TODO: make the label template configurable - label: format!("{} {:?} via {}", pk_membership.service_name, pk_membership.mode_name, via_name), - amount: pk_membership.payed_amount, - reference: answer.reference, - // TODO: make these field configurable - credit_account_code: "756".to_string(), // cotisations account - debit_account_code: debit_account_code.to_string(), // helloasso account - inception_time: answer.inception_time, - kind: paheko::TransactionKind::Revenue, - linked_users: pk_users_summaries.iter().map(|x| x.id.clone()).collect(), - // the linked_services, depend on a patch to paheko API code to work (see https://forge.lefuturiste.fr/mbess/paheko-fork/commit/a4fdd816112f51db23a2b02ac160b0513a5b09c5) - linked_subscriptions: pk_user_service_registrations.iter().map(|x| x.id.clone()).collect() - }; - let _ = paheko_client.register_transaction(transaction) - .await.context("Expected to create new paheko transaction")?; - eprintln!(" Created paheko transaction"); + if !existing_matching_transactions.iter().any(|t| t.credit_account_code == SUBSCRIPTION_ACCOUNT_CODE) { + // add transaction for subscription + let transaction = paheko::SimpleTransaction { + accounting_year: match get_accounting_year_for_time(&accounting_years, &answer.inception_time) { + None => { + eprintln!("Cannot find an accounting year on paheko that include the inception date {:?} given", &answer.inception_time); + panic!(); + }, + Some(s) => s + }.id.clone(), + // TODO: make the label template configurable + label: format!("{} {:?} via {}", pk_membership.service_name, pk_membership.mode_name, via_name), + amount: pk_membership.payed_amount, + reference: answer.reference.clone(), + // TODO: make these field configurable + credit_account_code: "756".to_string(), // cotisations account + debit_account_code: debit_account_code.to_string(), // helloasso account + inception_time: answer.inception_time, + kind: paheko::TransactionKind::Revenue, + linked_users: pk_users_summaries.iter().map(|x| x.id.clone()).collect(), + // the linked_services, depend on a patch to paheko API code to work (see https://forge.lefuturiste.fr/mbess/paheko-fork/commit/a4fdd816112f51db23a2b02ac160b0513a5b09c5) + linked_subscriptions: pk_user_service_registrations.iter().map(|x| x.id.clone()).collect() + }; + let _ = paheko_client.register_transaction(transaction) + .await.context("Expected to create new paheko transaction")?; + stats.transaction_created += 1; + eprintln!(" Created paheko transaction for subscription"); + } else { + eprintln!(" Skipped creation of paheko transaction for subscription"); + } + + // check if donation is already reference + // look for an existing donation regisered in DONATION_ACCOUNT_CODE with the same ref + if answer.donation_amount > 0.0 { + if !existing_matching_transactions.iter().any(|t| t.credit_account_code == DONATION_ACCOUNT_CODE) { + // add transaction for donation + let transaction = paheko::SimpleTransaction { + accounting_year: match get_accounting_year_for_time(&accounting_years, &answer.inception_time) { + None => { + eprintln!("Cannot find an accounting year on paheko that include the inception date {:?} given", &answer.inception_time); + panic!(); + }, + Some(s) => s + }.id.clone(), + label: format!("Don lié à une adhésion via {}", via_name), + amount: answer.donation_amount, + reference: answer.reference.clone(), + credit_account_code: DONATION_ACCOUNT_CODE.to_string(), // account 754 - Ressources liées à la générosité du public + debit_account_code: debit_account_code.to_string(), // compte d'encaissement + inception_time: answer.inception_time, + kind: paheko::TransactionKind::Revenue, + linked_users: pk_users_summaries.iter().map(|x| x.id.clone()).collect(), + linked_subscriptions: vec![] + }; + let _ = paheko_client.register_transaction(transaction) + .await.context("Expected to create new paheko transaction for donation")?; + stats.transaction_created += 1; + eprintln!(" Created paheko transaction for donation"); + } else { + eprintln!(" Skipped creation of transaction donation"); + } + } - stats.subscriptions_created += 1; // TODO: handle donation amount pk_memberships.push(pk_membership); } eprintln!("{via_name} sync done."); - eprintln!("{} subs created; {} users created", stats.subscriptions_created, stats.users_created); + eprintln!("{} subs created; {} users created; {} transactions created", stats.subscriptions_created, stats.users_created, stats.transaction_created); Ok(()) } diff --git a/src/utils.rs b/src/utils.rs index 006b478..4420c71 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -23,27 +23,6 @@ pub fn generate_id() -> Id { } -/// https://serde.rs/field-attrs.html -pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - DateTime::parse_from_rfc3339(&s) - .map_err(serde::de::Error::custom) - .map(|dt| dt.with_timezone(&Utc)) -} - -pub fn deserialize_date<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - NaiveDate::parse_from_str(&s, "%Y-%m-%d") - .map_err(serde::de::Error::custom) -} - - pub fn parse_datetime(inp: &str) -> Option> { let date = NaiveDate::parse_from_str(inp, "%d/%m/%Y").ok()?; Some(DateTime::::from_naive_utc_and_offset( @@ -73,6 +52,12 @@ pub fn parse_normalize_phone(inp: String) -> Option { Some(parsed.to_string()) } +pub fn complete_date(inp: NaiveDate) -> DateTime { + DateTime::::from_naive_utc_and_offset( + inp.and_hms_opt(0, 0, 0).unwrap(), + Utc + ) +} pub fn normalize_str(subject: String) -> String { subject.trim().replace("\n", ";").to_string() @@ -108,3 +93,32 @@ pub fn normalize_last_name(subject: String) -> String { .to_uppercase() } + + +/// https://serde.rs/field-attrs.html +pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + DateTime::parse_from_rfc3339(&s) + .map_err(serde::de::Error::custom) + .map(|dt| dt.with_timezone(&Utc)) +} + +pub fn deserialize_date<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + NaiveDate::parse_from_str(&s, "%Y-%m-%d") + .map_err(serde::de::Error::custom) +} + +pub fn deserialize_json_list<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + serde_json::from_str(&s).map_err(serde::de::Error::custom) +} diff --git a/sync_csv.sh b/sync_csv.sh new file mode 100755 index 0000000..c8332f0 --- /dev/null +++ b/sync_csv.sh @@ -0,0 +1,7 @@ +#!/usr/bin/sh + +xlsx2csv -n "Adhérents PAPIER" /warmd/etoiledebethleem/copyparty/adhesions/wip_matthieu_b.xlsx \ + | sed ':a;N;$!ba;s/\(Champ complémentaire [0-9]\)\n/\1 /g' \ + | sed 's/Champ complémentaire \([0-9]\)/CC \1/g' \ + | cargo run -- --source csv +