feat: handle additional donations

TODO.md

@@ -53,3 +53,8 @@ TODO:
 query all subscriptions of user byu service label
 
 curl -u $PAHEKO_CLIENT_ID:$PAHEKO_CLIENT_SECRET http://localhost:8082/api/sql -d "SELECT su.id_user,su.date FROM services_users AS su JOIN services AS s ON su.id_service = s.id WHERE s.label = 'Cotisation 2023-2024';"
+
+## 2024-01-20
+
+- handle additional donation amount
+

src/helloasso.rs

@@ -310,7 +310,10 @@ struct Payment {
     id: u64,
 
     #[serde(rename = "type")]
-    extra: Option<String>
+    extra: Option<String>,
+
+    share_amount: u32,
+    amount: u32
 }
 
 #[derive(Debug, Serialize, Deserialize)]

src/main.rs

@@ -17,7 +17,6 @@ use serde::{Serialize, Deserialize};
 use url::Url;
 use fully_pub::fully_pub;
 use argh::FromArgs;
-use utils::{parse_normalize_phone, normalize_str, parse_and_get_birthday_year};
 
 /// permanent config to store long-term config
 /// used to ingest env settings
@@ -38,7 +37,7 @@ struct Config {
     paheko_client_secret: String,
 
     paheko_target_activity_name: String,
-    // paheko_accounting_year_id: u64,
+    paheko_accounting_years_ids: Vec<u32>,
 }
 
 // start user cache management
@@ -137,10 +136,7 @@ fn get_proxy_from_url(proxy_url: &Option<String>) -> Result<Option<reqwest::Prox
     })
 }
 
-async fn launch_adapter(source: SourceType) -> Result<()> {
+async fn launch_adapter(source: SourceType, config: &Config) -> Result<()> {
-    dotenvy::dotenv()?;
-
-    let config: Config = envy::from_env().context("Failed to load env vars")?;
 
     let mut user_cache = load_user_cache().context("Failed to load user cache")?;
 
@@ -173,7 +169,11 @@ async fn launch_adapter(source: SourceType) -> Result<()> {
 struct App {
     /// the source of sync (CSV or helloasso)
     #[argh(option, short = 'm')]
-    source: String,
+    source: Option<String>,
+
+    /// output debug info
+    #[argh(switch, short = 'i')]
+    info: bool
 }
 
 enum SourceType {
@@ -184,7 +184,15 @@ enum SourceType {
 #[tokio::main]
 async fn main() {
     let app: App = argh::from_env();
-    let source = match app.source.as_ref() {
+    dotenvy::dotenv().expect("Could not load dot env file");
+    let config: Config = envy::from_env().expect("Failed to load env vars");
+
+    if app.info {
+        dbg!(config);
+        return;
+    }
+
+    let source = match app.source.unwrap().as_ref() {
         "helloasso" => SourceType::Helloasso,
         "csv" => SourceType::Csv,
         _ => {
@@ -192,7 +200,7 @@ async fn main() {
             return;
         }
     };
-    let res = launch_adapter(source).await;
+    let res = launch_adapter(source, &config).await;
     match res {
         Err(err) => {
             eprintln!("Program failed, details bellow");

src/paheko.rs

@@ -8,7 +8,7 @@ use chrono::prelude::{DateTime, Utc};
 use chrono::NaiveDate;
 use thiserror::Error;
 use crate::sync_paheko::GeneralizedAnswer;
-use crate::utils::deserialize_date;
+use crate::utils::{deserialize_date, deserialize_json_list, complete_date};
 
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
@@ -250,13 +250,6 @@ struct SqlQueryOutput {
     results: serde_json::Value
 }
 
-#[derive(Debug, Deserialize)]
-#[fully_pub]
-struct TransactionSummary {
-    id: u64,
-    reference: String
-}
-
 #[derive(Debug)]
 #[fully_pub]
 struct UserServiceRegistration {
@@ -320,16 +313,53 @@ impl AuthentifiedClient {
         Ok(ids.get(0).map(|x| x.id).unwrap_or(1)+1)
     }
 
-    pub async fn get_transactions(&self, id_year: u32)
+    pub async fn get_transactions(&self, id_years: &Vec<u32>)
-        -> Result<Vec<TransactionSummary>>
+        -> Result<Vec<SimpleTransaction>>
     {
+        #[derive(Debug, Deserialize)]
+        struct Row {
+            id: u64,
+            label: String,
+            reference: String,
+            #[serde(deserialize_with = "deserialize_json_list")]
+            accounts_codes: Vec<String>,
+            year_id: u64,
+            #[serde(deserialize_with = "deserialize_date")]
+            inception_date: NaiveDate
+        }
+
+        let id_years_joined = id_years.iter()
+            .map(|x| x.to_string())
+            .collect::<Vec<String>>()
+            .join(",");
         let query: String = format!(r#"
-            SELECT id,reference FROM acc_transactions WHERE id_year={} AND reference LIKE 'HA/%';
+            SELECT act.id, act.date AS inception_date, act.id_year AS year_id, act.label, act.reference, acc.code, JSON_GROUP_ARRAY(acc.code) AS accounts_codes
-        "#, id_year).to_string();
+            FROM acc_transactions AS act
+            INNER JOIN acc_transactions_lines AS actl ON actl.id_transaction = act.id
+            INNER JOIN acc_accounts AS acc ON acc.id = actl.id_account
+            WHERE act.id_year IN ({})
+            GROUP BY act.id;
+        "#, id_years_joined).to_string();
 
         let val = self.sql_query(query).await.context("Fetching transactions")?;
         
-        Ok(serde_json::from_value(val.results)?)
+        let raw_vals: Vec<Row> = serde_json::from_value(val.results)
+            .context("Cannot deserialize SQL transactions rows")?;
+
+        // we will assume that the first acc code is always the credit, and second the debit
+
+        Ok(raw_vals.iter().map(|x| SimpleTransaction {
+            label: x.label.clone(),
+            reference: x.reference.clone(),
+            credit_account_code: x.accounts_codes.get(0).unwrap().to_string(),
+            debit_account_code: x.accounts_codes.get(1).unwrap().to_string(),
+            inception_time: complete_date(x.inception_date),
+            accounting_year: Id(x.year_id),
+            amount: 0.0,
+            kind: TransactionKind::Expense,
+            linked_subscriptions: vec![],
+            linked_users: vec![],
+        }).collect())
     }
 
     pub async fn get_accounting_years(&self)

src/sync_csv.rs

@@ -13,6 +13,8 @@ use csv::ReaderBuilder;
 
 use std::io;
 
+const CAISSE_ACCOUNT_CODE: &str = "530"; // 530 - Caisse
+
 fn process_csv_value(value: String) -> Option<String> {
     let value = normalize_str(value);
     if value.is_empty() {
@@ -80,6 +82,7 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi
     let mut intermediate_inp = "".to_string();
     for line_res in stdin.lock().lines() {
         let line = line_res.unwrap();
+        eprintln!("{:?}",&line);
         if line.starts_with(",") {
             continue;
         }
@@ -95,9 +98,11 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi
 
     let mut generalized_answers: Vec<GeneralizedAnswer> = vec![];
 
+    eprintln!("Reading from stdin");
+
     for parsed_record_res in rdr.deserialize() {
         let parsed_record: AnswerRecord = parsed_record_res?;
-        println!("parsed_record: {:?}", parsed_record);
+        eprintln!("Parsed_record: {:?}", parsed_record);
 
         let generalized_answer = GeneralizedAnswer {
             first_name: Some(normalize_str(parsed_record.first_name)),
@@ -131,13 +136,13 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi
 
         generalized_answers.push(generalized_answer);
     }
-    println!("Generated GeneralizedAnswers");
+    eprintln!("Generated GeneralizedAnswers");
     sync_paheko(
         paheko_client,
         config,
         user_cache,
         generalized_answers,
-        "530",
+        CAISSE_ACCOUNT_CODE,
         "Papier"
     ).await?;
     eprintln!("CSV sync done.");

src/sync_helloasso.rs

@@ -89,11 +89,13 @@ pub async fn sync_helloasso(paheko_client: &paheko::AuthentifiedClient, config:
         let email = choose_email(&answer);
 
         // skip answers that were imported later and are stranger from helloasso
-        if let Some(payment) = answer.payments.iter().nth(0) {
+        let payment = answer.payments.iter().nth(0).expect("Expected payment to exists");
         if payment.extra == Some("Offline".to_string()) {
             continue;
         }
-        }
+
+        let subscription_amount = f64::from(payment.share_amount)/100.0;
+        let donation_amount = f64::from(payment.amount - payment.share_amount)/100.0;
 
         let mut generalized_answer = GeneralizedAnswer {
             first_name: Some(normalize_str(answer.user.first_name.clone())),
@@ -117,8 +119,10 @@ pub async fn sync_helloasso(paheko_client: &paheko::AuthentifiedClient, config:
             birth_year: read_custom_field(&answer, HelloassoCustomFieldType::Birthday).and_then(parse_and_get_birthday_year),
             inception_time: answer.order.inception_time,
             reference: format!("HA/{}", answer.id),
-            donation_amount: 0.0,
+            // TODO: handle donation from helloasso, compare initial_amount and amount in payment
-            subscription_amount: f64::from(answer.amount)/100.0,
+            // or shareAmount
+            donation_amount,
+            subscription_amount,
             membership_mode: serde_json::from_value(serde_json::Value::String(answer.mode.clone()))
                 .expect("Expected a membership mode to be valid"),
             linked_user_first_name: read_custom_field(&answer, HelloassoCustomFieldType::LinkedUserFirstName)

src/sync_paheko.rs

@@ -1,5 +1,5 @@
 use crate::paheko;
-use crate::paheko::AccountingYear;
+use crate::paheko::{AccountingYear, SimpleTransaction};
 use crate::{
     Config, UserCache,
 };
@@ -10,6 +10,9 @@ use chrono::prelude::{NaiveDate, DateTime, Utc};
 use fully_pub::fully_pub;
 use serde::{Serialize, Deserialize};
 
+const DONATION_ACCOUNT_CODE: &str = "754";
+const SUBSCRIPTION_ACCOUNT_CODE: &str = "756";
+
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
 #[fully_pub]
 enum MembershipMode {
@@ -67,10 +70,11 @@ pub async fn sync_paheko(
     
     struct Stats {
         subscriptions_created: u32,
+        transaction_created: u32,
         users_created: u32
     }
 
-    let mut stats = Stats { subscriptions_created: 0, users_created: 0 };
+    let mut stats = Stats { transaction_created: 0, subscriptions_created: 0, users_created: 0 };
 
     let mut pk_memberships: Vec<paheko::Membership> = vec![];
 
@@ -80,7 +84,8 @@ pub async fn sync_paheko(
     let mut existing_users = paheko_client.get_users().await.context("Get users")?;
     // 2. get summary of transactions for that year
     // TODO: also get for the year n-1
-    let existing_transactions = paheko_client.get_transactions(1).await.context("Get transactions")?;
+    let existing_transactions = paheko_client.get_transactions(&config.paheko_accounting_years_ids)
+        .await.context("Get transactions")?;
     // 3. get summary of services_users for that year
     let existing_subscriptions = paheko_client.get_service_subscriptions(&config.paheko_target_activity_name)
         .await.context("Get existing paheko subscriptions to the target activity")?;
@@ -106,24 +111,21 @@ pub async fn sync_paheko(
         let mut pk_users_summaries: Vec<paheko::UserSummary> = vec![];
         let mut pk_user_service_registrations: Vec<paheko::UserServiceRegistration> = vec![];
 
+
+        let existing_matching_transactions: Vec<&SimpleTransaction> = existing_transactions
+            .iter()
+            .filter(|t| t.reference == answer.reference)
+            .collect();
+
+        // dbg!(&existing_subscriptions);
         // check for existing user in paheko by email
         // TODO: check user with fuzzing first name and last name
         let existing_user_opt = existing_users
             .iter().find(|user| user.first_name == answer.first_name && user.last_name == answer.last_name)
             .cloned();
-
-        // check for existing transactions
-        if existing_transactions.iter().any(
-            |summary| summary.reference == answer.reference
-        ) {
-            eprintln!(" skipped: existing transaction found");
-            continue;
-        }
-
-        // dbg!(&existing_subscriptions);
         let pk_user_summary = match existing_user_opt.clone() {
             Some(user) => {
-                eprintln!(" Found existing paheko user by name.");
+                eprintln!(" Found existing paheko user by matching name.");
                 user
             },
             None => {
@@ -140,18 +142,6 @@ pub async fn sync_paheko(
         };
         pk_users_summaries.push(pk_user_summary.clone());
 
-        // check if the user is already subscribed to the target activity
-        if 
-            existing_user_opt.is_some() &&
-            existing_subscriptions.iter()
-                .find(|membership| membership.users_ids
-                    .iter().find(|i| **i == pk_user_summary.id).is_some()
-                )
-                .is_some()
-        {
-            eprintln!(" skipped: user is already subscribed to this activity");
-            continue;
-        }
         
         let mut pk_membership = paheko::Membership {
             id: generate_id(),
@@ -168,7 +158,16 @@ pub async fn sync_paheko(
             payed_amount: answer.subscription_amount,
             users_ids: vec![pk_user_summary.id.clone()]
         };
-
+        // check if the user is already subscribed to the target activity
+        if 
+            existing_user_opt.is_some() &&
+            existing_subscriptions.iter()
+                .find(|membership| membership.users_ids
+                    .iter().find(|i| **i == pk_user_summary.id).is_some()
+                )
+                .is_some() {
+                eprintln!(" User is already subscribed to this activity");
+        } else {
             // add activity for first member
             // TODO: check if activity already exists
             let user_registration = paheko_client.register_user_to_service(
@@ -178,9 +177,12 @@ pub async fn sync_paheko(
             ).await.context("Expected to register user activity to paheko")?;
             pk_user_service_registrations.push(user_registration);
             pk_next_user_service_id += 1;
+            stats.subscriptions_created += 1;
             eprintln!(" Created paheko activity registration");
 
             // then create optional linked user
+            // FIXME: reuse a previous user
+            // TODO: get existing linked user from previous year
             if answer.membership_mode == MembershipMode::Couple {
                 let mut second_answer = answer.clone();
                 second_answer.email = None;
@@ -216,16 +218,16 @@ pub async fn sync_paheko(
                     ).await.context("Registering service to second paheko server")?;
                     pk_user_service_registrations.push(user_registration);
                     pk_next_user_service_id += 1;
+                    stats.subscriptions_created += 1;
                     eprintln!(" Created paheko activity registration for conjoint user");
 
                     pk_membership.users_ids.push(second_pk_user_summary.id)
                 }
-            // FIXME: reuse a previous user
+            }
-            // TODO: get existing linked user from previous year
-
         }
 
-        // add transaction
+        if !existing_matching_transactions.iter().any(|t| t.credit_account_code == SUBSCRIPTION_ACCOUNT_CODE) {
+            // add transaction for subscription
             let transaction = paheko::SimpleTransaction {
                 accounting_year: match get_accounting_year_for_time(&accounting_years, &answer.inception_time) {
                     None => {
@@ -237,7 +239,7 @@ pub async fn sync_paheko(
                 // TODO: make the label template configurable
                 label: format!("{} {:?} via {}", pk_membership.service_name, pk_membership.mode_name, via_name),
                 amount: pk_membership.payed_amount,
-            reference: answer.reference,
+                reference: answer.reference.clone(),
                 // TODO: make these field configurable
                 credit_account_code: "756".to_string(), // cotisations account
                 debit_account_code: debit_account_code.to_string(), // helloasso account
@@ -249,15 +251,50 @@ pub async fn sync_paheko(
             };
             let _ = paheko_client.register_transaction(transaction)
                 .await.context("Expected to create new paheko transaction")?;
-        eprintln!(" Created paheko transaction");
+            stats.transaction_created += 1;
+            eprintln!(" Created paheko transaction for subscription");
+        } else {
+            eprintln!(" Skipped creation of paheko transaction for subscription");
+        }
+        
+        // check if donation is already reference
+        // look for an existing donation regisered in DONATION_ACCOUNT_CODE with the same ref
+        if answer.donation_amount > 0.0 {
+            if !existing_matching_transactions.iter().any(|t| t.credit_account_code == DONATION_ACCOUNT_CODE) {
+                // add transaction for donation
+                let transaction = paheko::SimpleTransaction {
+                    accounting_year: match get_accounting_year_for_time(&accounting_years, &answer.inception_time) {
+                        None => {
+                            eprintln!("Cannot find an accounting year on paheko that include the inception date {:?} given", &answer.inception_time);
+                            panic!();
+                        },
+                        Some(s) => s
+                    }.id.clone(),
+                    label: format!("Don lié à une adhésion via {}", via_name),
+                    amount: answer.donation_amount,
+                    reference: answer.reference.clone(),
+                    credit_account_code: DONATION_ACCOUNT_CODE.to_string(), // account  754 - Ressources liées à la générosité du public
+                    debit_account_code: debit_account_code.to_string(), // compte d'encaissement
+                    inception_time: answer.inception_time,
+                    kind: paheko::TransactionKind::Revenue,
+                    linked_users: pk_users_summaries.iter().map(|x| x.id.clone()).collect(),
+                    linked_subscriptions: vec![]
+                };
+                let _ = paheko_client.register_transaction(transaction)
+                    .await.context("Expected to create new paheko transaction for donation")?;
+                stats.transaction_created += 1;
+                eprintln!(" Created paheko transaction for donation");
+            } else {
+                eprintln!(" Skipped creation of transaction donation");
+            }
+        }
 
-        stats.subscriptions_created += 1;
 
         // TODO: handle donation amount
 
         pk_memberships.push(pk_membership);
     }
     eprintln!("{via_name} sync done.");
-    eprintln!("{} subs created; {} users created", stats.subscriptions_created, stats.users_created);
+    eprintln!("{} subs created; {} users created; {} transactions created", stats.subscriptions_created, stats.users_created, stats.transaction_created);
     Ok(())
 }

src/utils.rs

@@ -23,27 +23,6 @@ pub fn generate_id() -> Id {
 }
 
 
-/// https://serde.rs/field-attrs.html
-pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result<DateTime<Utc>, D::Error>
-where
-    D: Deserializer<'de>,
-{
-    let s = String::deserialize(deserializer)?;
-    DateTime::parse_from_rfc3339(&s)
-        .map_err(serde::de::Error::custom)
-        .map(|dt| dt.with_timezone(&Utc))
-}
-
-pub fn deserialize_date<'de, D>(deserializer: D) -> Result<NaiveDate, D::Error>
-where
-    D: Deserializer<'de>,
-{
-    let s = String::deserialize(deserializer)?;
-    NaiveDate::parse_from_str(&s, "%Y-%m-%d")
-        .map_err(serde::de::Error::custom)
-}
-
-
 pub fn parse_datetime(inp: &str) -> Option<DateTime<Utc>> {
     let date = NaiveDate::parse_from_str(inp, "%d/%m/%Y").ok()?;
     Some(DateTime::<Utc>::from_naive_utc_and_offset(
@@ -73,6 +52,12 @@ pub fn parse_normalize_phone(inp: String) -> Option<String> {
     Some(parsed.to_string())
 }
 
+pub fn complete_date(inp: NaiveDate) -> DateTime<Utc> {
+    DateTime::<Utc>::from_naive_utc_and_offset(
+        inp.and_hms_opt(0, 0, 0).unwrap(),
+        Utc
+    )
+}
 
 pub fn normalize_str(subject: String) -> String {
     subject.trim().replace("\n", ";").to_string()
@@ -108,3 +93,32 @@ pub fn normalize_last_name(subject: String) -> String {
         .to_uppercase()
 }
 
+
+
+/// https://serde.rs/field-attrs.html
+pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result<DateTime<Utc>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s = String::deserialize(deserializer)?;
+    DateTime::parse_from_rfc3339(&s)
+        .map_err(serde::de::Error::custom)
+        .map(|dt| dt.with_timezone(&Utc))
+}
+
+pub fn deserialize_date<'de, D>(deserializer: D) -> Result<NaiveDate, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s = String::deserialize(deserializer)?;
+    NaiveDate::parse_from_str(&s, "%Y-%m-%d")
+        .map_err(serde::de::Error::custom)
+}
+
+pub fn deserialize_json_list<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s = String::deserialize(deserializer)?;
+    serde_json::from_str(&s).map_err(serde::de::Error::custom)
+}

sync_csv.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/sh
+
+xlsx2csv -n "Adhérents PAPIER" /warmd/etoiledebethleem/copyparty/adhesions/wip_matthieu_b.xlsx \
+    | sed ':a;N;$!ba;s/\(Champ complémentaire [0-9]\)\n/\1 /g' \
+    | sed 's/Champ complémentaire \([0-9]\)/CC \1/g' \
+    | cargo run -- --source csv
+