mbess/helloasso-paheko-adapter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

feat: handle additional donations

Browse source
This commit is contained in:
Matthieu Bessat 2024-01-21 14:37:30 +01:00
parent 52c7da4836
commit edb5d6a372
9 changed files with 271 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
TODO.md
View file

@ -53,3 +53,8 @@ TODO:
query all subscriptions of user byu service label
curl -u $PAHEKO_CLIENT_ID:$PAHEKO_CLIENT_SECRET http://localhost:8082/api/sql -d "SELECT su.id_user,su.date FROM services_users AS su JOIN services AS s ON su.id_service = s.id WHERE s.label = 'Cotisation 2023-2024';"
## 2024-01-20
- handle additional donation amount

5
src/helloasso.rs
View file

@ -310,7 +310,10 @@ struct Payment {
id: u64,
#[serde(rename = "type")]
extra: Option<String>
extra: Option<String>,
share_amount: u32,
amount: u32
}
#[derive(Debug, Serialize, Deserialize)]

26
src/main.rs
View file

@ -17,7 +17,6 @@ use serde::{Serialize, Deserialize};
use url::Url;
use fully_pub::fully_pub;
use argh::FromArgs;
use utils::{parse_normalize_phone, normalize_str, parse_and_get_birthday_year};
/// permanent config to store long-term config
/// used to ingest env settings
@ -38,7 +37,7 @@ struct Config {
paheko_client_secret: String,
paheko_target_activity_name: String,
// paheko_accounting_year_id: u64,
paheko_accounting_years_ids: Vec<u32>,
}
// start user cache management
@ -137,10 +136,7 @@ fn get_proxy_from_url(proxy_url: &Option<String>) -> Result<Option<reqwest::Prox
})
}
async fn launch_adapter(source: SourceType) -> Result<()> {
dotenvy::dotenv()?;
let config: Config = envy::from_env().context("Failed to load env vars")?;
async fn launch_adapter(source: SourceType, config: &Config) -> Result<()> {
let mut user_cache = load_user_cache().context("Failed to load user cache")?;
@ -173,7 +169,11 @@ async fn launch_adapter(source: SourceType) -> Result<()> {
struct App {
/// the source of sync (CSV or helloasso)
#[argh(option, short = 'm')]
source: String,
source: Option<String>,
/// output debug info
#[argh(switch, short = 'i')]
info: bool
}
enum SourceType {
@ -184,7 +184,15 @@ enum SourceType {
#[tokio::main]
async fn main() {
let app: App = argh::from_env();
let source = match app.source.as_ref() {
dotenvy::dotenv().expect("Could not load dot env file");
let config: Config = envy::from_env().expect("Failed to load env vars");
if app.info {
dbg!(config);
return;
}
let source = match app.source.unwrap().as_ref() {
"helloasso" => SourceType::Helloasso,
"csv" => SourceType::Csv,
_ => {
@ -192,7 +200,7 @@ async fn main() {
return;
}
};
let res = launch_adapter(source).await;
let res = launch_adapter(source, &config).await;
match res {
Err(err) => {
eprintln!("Program failed, details bellow");

56
src/paheko.rs
View file

@ -8,7 +8,7 @@ use chrono::prelude::{DateTime, Utc};
use chrono::NaiveDate;
use thiserror::Error;
use crate::sync_paheko::GeneralizedAnswer;
use crate::utils::deserialize_date;
use crate::utils::{deserialize_date, deserialize_json_list, complete_date};
#[derive(Debug, Serialize, Deserialize, Clone)]
@ -250,13 +250,6 @@ struct SqlQueryOutput {
results: serde_json::Value
}
#[derive(Debug, Deserialize)]
#[fully_pub]
struct TransactionSummary {
id: u64,
reference: String
}
#[derive(Debug)]
#[fully_pub]
struct UserServiceRegistration {
@ -320,16 +313,53 @@ impl AuthentifiedClient {
Ok(ids.get(0).map(|x| x.id).unwrap_or(1)+1)
}
pub async fn get_transactions(&self, id_year: u32)
-> Result<Vec<TransactionSummary>>
pub async fn get_transactions(&self, id_years: &Vec<u32>)
-> Result<Vec<SimpleTransaction>>
{
#[derive(Debug, Deserialize)]
struct Row {
id: u64,
label: String,
reference: String,
#[serde(deserialize_with = "deserialize_json_list")]
accounts_codes: Vec<String>,
year_id: u64,
#[serde(deserialize_with = "deserialize_date")]
inception_date: NaiveDate
}
let id_years_joined = id_years.iter()
.map(|x| x.to_string())
.collect::<Vec<String>>()
.join(",");
let query: String = format!(r#"
SELECT id,reference FROM acc_transactions WHERE id_year={} AND reference LIKE 'HA/%';
"#, id_year).to_string();
SELECT act.id, act.date AS inception_date, act.id_year AS year_id, act.label, act.reference, acc.code, JSON_GROUP_ARRAY(acc.code) AS accounts_codes
FROM acc_transactions AS act
INNER JOIN acc_transactions_lines AS actl ON actl.id_transaction = act.id
INNER JOIN acc_accounts AS acc ON acc.id = actl.id_account
WHERE act.id_year IN ({})
GROUP BY act.id;
"#, id_years_joined).to_string();
let val = self.sql_query(query).await.context("Fetching transactions")?;
Ok(serde_json::from_value(val.results)?)
let raw_vals: Vec<Row> = serde_json::from_value(val.results)
.context("Cannot deserialize SQL transactions rows")?;
// we will assume that the first acc code is always the credit, and second the debit
Ok(raw_vals.iter().map(|x| SimpleTransaction {
label: x.label.clone(),
reference: x.reference.clone(),
credit_account_code: x.accounts_codes.get(0).unwrap().to_string(),
debit_account_code: x.accounts_codes.get(1).unwrap().to_string(),
inception_time: complete_date(x.inception_date),
accounting_year: Id(x.year_id),
amount: 0.0,
kind: TransactionKind::Expense,
linked_subscriptions: vec![],
linked_users: vec![],
}).collect())
}
pub async fn get_accounting_years(&self)

11
src/sync_csv.rs
View file

@ -13,6 +13,8 @@ use csv::ReaderBuilder;
use std::io;
const CAISSE_ACCOUNT_CODE: &str = "530"; // 530 - Caisse
fn process_csv_value(value: String) -> Option<String> {
let value = normalize_str(value);
if value.is_empty() {
@ -80,6 +82,7 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi
let mut intermediate_inp = "".to_string();
for line_res in stdin.lock().lines() {
let line = line_res.unwrap();
eprintln!("{:?}",&line);
if line.starts_with(",") {
continue;
}
@ -95,9 +98,11 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi
let mut generalized_answers: Vec<GeneralizedAnswer> = vec![];
eprintln!("Reading from stdin");
for parsed_record_res in rdr.deserialize() {
let parsed_record: AnswerRecord = parsed_record_res?;
println!("parsed_record: {:?}", parsed_record);
eprintln!("Parsed_record: {:?}", parsed_record);
let generalized_answer = GeneralizedAnswer {
first_name: Some(normalize_str(parsed_record.first_name)),
@ -131,13 +136,13 @@ pub async fn sync_csv(paheko_client: &paheko::AuthentifiedClient, config: &Confi
generalized_answers.push(generalized_answer);
}
println!("Generated GeneralizedAnswers");
eprintln!("Generated GeneralizedAnswers");
sync_paheko(
paheko_client,
config,
user_cache,
generalized_answers,
"530",
CAISSE_ACCOUNT_CODE,
"Papier"
).await?;
eprintln!("CSV sync done.");

16
src/sync_helloasso.rs
View file

@ -89,12 +89,14 @@ pub async fn sync_helloasso(paheko_client: &paheko::AuthentifiedClient, config:
let email = choose_email(&answer);
// skip answers that were imported later and are stranger from helloasso
if let Some(payment) = answer.payments.iter().nth(0) {
if payment.extra == Some("Offline".to_string()) {
continue;
}
let payment = answer.payments.iter().nth(0).expect("Expected payment to exists");
if payment.extra == Some("Offline".to_string()) {
continue;
}
let subscription_amount = f64::from(payment.share_amount)/100.0;
let donation_amount = f64::from(payment.amount - payment.share_amount)/100.0;
let mut generalized_answer = GeneralizedAnswer {
first_name: Some(normalize_str(answer.user.first_name.clone())),
last_name: normalize_str(answer.user.last_name.clone()),
@ -117,8 +119,10 @@ pub async fn sync_helloasso(paheko_client: &paheko::AuthentifiedClient, config:
birth_year: read_custom_field(&answer, HelloassoCustomFieldType::Birthday).and_then(parse_and_get_birthday_year),
inception_time: answer.order.inception_time,
reference: format!("HA/{}", answer.id),
donation_amount: 0.0,
subscription_amount: f64::from(answer.amount)/100.0,
// TODO: handle donation from helloasso, compare initial_amount and amount in payment
// or shareAmount
donation_amount,
subscription_amount,
membership_mode: serde_json::from_value(serde_json::Value::String(answer.mode.clone()))
.expect("Expected a membership mode to be valid"),
linked_user_first_name: read_custom_field(&answer, HelloassoCustomFieldType::LinkedUserFirstName)

247
src/sync_paheko.rs
View file

@ -1,5 +1,5 @@
use crate::paheko;
use crate::paheko::AccountingYear;
use crate::paheko::{AccountingYear, SimpleTransaction};
use crate::{
Config, UserCache,
};
@ -10,6 +10,9 @@ use chrono::prelude::{NaiveDate, DateTime, Utc};
use fully_pub::fully_pub;
use serde::{Serialize, Deserialize};
const DONATION_ACCOUNT_CODE: &str = "754";
const SUBSCRIPTION_ACCOUNT_CODE: &str = "756";
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
#[fully_pub]
enum MembershipMode {
@ -67,10 +70,11 @@ pub async fn sync_paheko(
struct Stats {
subscriptions_created: u32,
transaction_created: u32,
users_created: u32
}
let mut stats = Stats { subscriptions_created: 0, users_created: 0 };
let mut stats = Stats { transaction_created: 0, subscriptions_created: 0, users_created: 0 };
let mut pk_memberships: Vec<paheko::Membership> = vec![];
@ -80,7 +84,8 @@ pub async fn sync_paheko(
let mut existing_users = paheko_client.get_users().await.context("Get users")?;
// 2. get summary of transactions for that year
// TODO: also get for the year n-1
let existing_transactions = paheko_client.get_transactions(1).await.context("Get transactions")?;
let existing_transactions = paheko_client.get_transactions(&config.paheko_accounting_years_ids)
.await.context("Get transactions")?;
// 3. get summary of services_users for that year
let existing_subscriptions = paheko_client.get_service_subscriptions(&config.paheko_target_activity_name)
.await.context("Get existing paheko subscriptions to the target activity")?;
@ -106,24 +111,21 @@ pub async fn sync_paheko(
let mut pk_users_summaries: Vec<paheko::UserSummary> = vec![];
let mut pk_user_service_registrations: Vec<paheko::UserServiceRegistration> = vec![];
let existing_matching_transactions: Vec<&SimpleTransaction> = existing_transactions
.iter()
.filter(|t| t.reference == answer.reference)
.collect();
// dbg!(&existing_subscriptions);
// check for existing user in paheko by email
// TODO: check user with fuzzing first name and last name
let existing_user_opt = existing_users
.iter().find(|user| user.first_name == answer.first_name && user.last_name == answer.last_name)
.cloned();
// check for existing transactions
if existing_transactions.iter().any(
|summary| summary.reference == answer.reference
) {
eprintln!(" skipped: existing transaction found");
continue;
}
// dbg!(&existing_subscriptions);
let pk_user_summary = match existing_user_opt.clone() {
Some(user) => {
eprintln!(" Found existing paheko user by name.");
eprintln!(" Found existing paheko user by matching name.");
user
},
None => {
@ -139,20 +141,8 @@ pub async fn sync_paheko(
}
};
pk_users_summaries.push(pk_user_summary.clone());
// check if the user is already subscribed to the target activity
if
existing_user_opt.is_some() &&
existing_subscriptions.iter()
.find(|membership| membership.users_ids
.iter().find(|i| **i == pk_user_summary.id).is_some()
)
.is_some()
{
eprintln!(" skipped: user is already subscribed to this activity");
continue;
}
let mut pk_membership = paheko::Membership {
id: generate_id(),
service_name: config.paheko_target_activity_name.clone(),
@ -168,96 +158,143 @@ pub async fn sync_paheko(
payed_amount: answer.subscription_amount,
users_ids: vec![pk_user_summary.id.clone()]
};
// check if the user is already subscribed to the target activity
if
existing_user_opt.is_some() &&
existing_subscriptions.iter()
.find(|membership| membership.users_ids
.iter().find(|i| **i == pk_user_summary.id).is_some()
)
.is_some() {
eprintln!(" User is already subscribed to this activity");
} else {
// add activity for first member
// TODO: check if activity already exists
let user_registration = paheko_client.register_user_to_service(
pk_users_summaries.get(0).unwrap(),
&pk_membership,
pk_next_user_service_id
).await.context("Expected to register user activity to paheko")?;
pk_user_service_registrations.push(user_registration);
pk_next_user_service_id += 1;
stats.subscriptions_created += 1;
eprintln!(" Created paheko activity registration");
// add activity for first member
// TODO: check if activity already exists
let user_registration = paheko_client.register_user_to_service(
pk_users_summaries.get(0).unwrap(),
&pk_membership,
pk_next_user_service_id
).await.context("Expected to register user activity to paheko")?;
pk_user_service_registrations.push(user_registration);
pk_next_user_service_id += 1;
eprintln!(" Created paheko activity registration");
// then create optional linked user
if answer.membership_mode == MembershipMode::Couple {
let mut second_answer = answer.clone();
second_answer.email = None;
second_answer.phone = None;
second_answer.skills = None;
second_answer.job = None;
second_answer.birth_year = None;
// add first_name
match answer.linked_user_first_name {
Some(name) => {
second_answer.first_name = Some(name);
},
None => {
second_answer.first_name = None;
eprintln!("Warn: Got a user with Couple mode but no additional name given!")
}
}
if existing_user_opt.is_none() {
// only create the linked user in paheko, if the first user was also created
let second_pk_user_summary = paheko_client.create_user(&second_answer, pk_next_user_id)
.await.context("Expected to create second paheko user")?;
eprintln!(" Created conjoint paheko user");
pk_users_summaries.push(second_pk_user_summary.clone());
pk_next_user_id += 1;
// create activity of second user
let user_registration = paheko_client.register_user_to_service(
pk_users_summaries.get(1).unwrap(), // pass user, for the id
&pk_membership,
pk_next_user_service_id
).await.context("Registering service to second paheko server")?;
pk_user_service_registrations.push(user_registration);
pk_next_user_service_id += 1;
eprintln!(" Created paheko activity registration for conjoint user");
pk_membership.users_ids.push(second_pk_user_summary.id)
}
// then create optional linked user
// FIXME: reuse a previous user
// TODO: get existing linked user from previous year
if answer.membership_mode == MembershipMode::Couple {
let mut second_answer = answer.clone();
second_answer.email = None;
second_answer.phone = None;
second_answer.skills = None;
second_answer.job = None;
second_answer.birth_year = None;
// add first_name
match answer.linked_user_first_name {
Some(name) => {
second_answer.first_name = Some(name);
},
None => {
second_answer.first_name = None;
eprintln!("Warn: Got a user with Couple mode but no additional name given!")
}
}
if existing_user_opt.is_none() {
// only create the linked user in paheko, if the first user was also created
let second_pk_user_summary = paheko_client.create_user(&second_answer, pk_next_user_id)
.await.context("Expected to create second paheko user")?;
eprintln!(" Created conjoint paheko user");
pk_users_summaries.push(second_pk_user_summary.clone());
pk_next_user_id += 1;
// create activity of second user
let user_registration = paheko_client.register_user_to_service(
pk_users_summaries.get(1).unwrap(), // pass user, for the id
&pk_membership,
pk_next_user_service_id
).await.context("Registering service to second paheko server")?;
pk_user_service_registrations.push(user_registration);
pk_next_user_service_id += 1;
stats.subscriptions_created += 1;
eprintln!(" Created paheko activity registration for conjoint user");
pk_membership.users_ids.push(second_pk_user_summary.id)
}
}
}
// add transaction
let transaction = paheko::SimpleTransaction {
accounting_year: match get_accounting_year_for_time(&accounting_years, &answer.inception_time) {
None => {
eprintln!("Cannot find an accounting year on paheko that include the inception date {:?} given", &answer.inception_time);
panic!();
},
Some(s) => s
}.id.clone(),
// TODO: make the label template configurable
label: format!("{} {:?} via {}", pk_membership.service_name, pk_membership.mode_name, via_name),
amount: pk_membership.payed_amount,
reference: answer.reference,
// TODO: make these field configurable
credit_account_code: "756".to_string(), // cotisations account
debit_account_code: debit_account_code.to_string(), // helloasso account
inception_time: answer.inception_time,
kind: paheko::TransactionKind::Revenue,
linked_users: pk_users_summaries.iter().map(|x| x.id.clone()).collect(),
// the linked_services, depend on a patch to paheko API code to work (see https://forge.lefuturiste.fr/mbess/paheko-fork/commit/a4fdd816112f51db23a2b02ac160b0513a5b09c5)
linked_subscriptions: pk_user_service_registrations.iter().map(|x| x.id.clone()).collect()
};
let _ = paheko_client.register_transaction(transaction)
.await.context("Expected to create new paheko transaction")?;
eprintln!(" Created paheko transaction");
if !existing_matching_transactions.iter().any(|t| t.credit_account_code == SUBSCRIPTION_ACCOUNT_CODE) {
// add transaction for subscription
let transaction = paheko::SimpleTransaction {
accounting_year: match get_accounting_year_for_time(&accounting_years, &answer.inception_time) {
None => {
eprintln!("Cannot find an accounting year on paheko that include the inception date {:?} given", &answer.inception_time);
panic!();
},
Some(s) => s
}.id.clone(),
// TODO: make the label template configurable
label: format!("{} {:?} via {}", pk_membership.service_name, pk_membership.mode_name, via_name),
amount: pk_membership.payed_amount,
reference: answer.reference.clone(),
// TODO: make these field configurable
credit_account_code: "756".to_string(), // cotisations account
debit_account_code: debit_account_code.to_string(), // helloasso account
inception_time: answer.inception_time,
kind: paheko::TransactionKind::Revenue,
linked_users: pk_users_summaries.iter().map(|x| x.id.clone()).collect(),
// the linked_services, depend on a patch to paheko API code to work (see https://forge.lefuturiste.fr/mbess/paheko-fork/commit/a4fdd816112f51db23a2b02ac160b0513a5b09c5)
linked_subscriptions: pk_user_service_registrations.iter().map(|x| x.id.clone()).collect()
};
let _ = paheko_client.register_transaction(transaction)
.await.context("Expected to create new paheko transaction")?;
stats.transaction_created += 1;
eprintln!(" Created paheko transaction for subscription");
} else {
eprintln!(" Skipped creation of paheko transaction for subscription");
}
// check if donation is already reference
// look for an existing donation regisered in DONATION_ACCOUNT_CODE with the same ref
if answer.donation_amount > 0.0 {
if !existing_matching_transactions.iter().any(|t| t.credit_account_code == DONATION_ACCOUNT_CODE) {
// add transaction for donation
let transaction = paheko::SimpleTransaction {
accounting_year: match get_accounting_year_for_time(&accounting_years, &answer.inception_time) {
None => {
eprintln!("Cannot find an accounting year on paheko that include the inception date {:?} given", &answer.inception_time);
panic!();
},
Some(s) => s
}.id.clone(),
label: format!("Don lié à une adhésion via {}", via_name),
amount: answer.donation_amount,
reference: answer.reference.clone(),
credit_account_code: DONATION_ACCOUNT_CODE.to_string(), // account 754 - Ressources liées à la générosité du public
debit_account_code: debit_account_code.to_string(), // compte d'encaissement
inception_time: answer.inception_time,
kind: paheko::TransactionKind::Revenue,
linked_users: pk_users_summaries.iter().map(|x| x.id.clone()).collect(),
linked_subscriptions: vec![]
};
let _ = paheko_client.register_transaction(transaction)
.await.context("Expected to create new paheko transaction for donation")?;
stats.transaction_created += 1;
eprintln!(" Created paheko transaction for donation");
} else {
eprintln!(" Skipped creation of transaction donation");
}
}
stats.subscriptions_created += 1;
// TODO: handle donation amount
pk_memberships.push(pk_membership);
}
eprintln!("{via_name} sync done.");
eprintln!("{} subs created; {} users created", stats.subscriptions_created, stats.users_created);
eprintln!("{} subs created; {} users created; {} transactions created", stats.subscriptions_created, stats.users_created, stats.transaction_created);
Ok(())
}

56
src/utils.rs
View file

@ -23,27 +23,6 @@ pub fn generate_id() -> Id {
}
/// https://serde.rs/field-attrs.html
pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result<DateTime<Utc>, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
DateTime::parse_from_rfc3339(&s)
.map_err(serde::de::Error::custom)
.map(|dt| dt.with_timezone(&Utc))
}
pub fn deserialize_date<'de, D>(deserializer: D) -> Result<NaiveDate, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
NaiveDate::parse_from_str(&s, "%Y-%m-%d")
.map_err(serde::de::Error::custom)
}
pub fn parse_datetime(inp: &str) -> Option<DateTime<Utc>> {
let date = NaiveDate::parse_from_str(inp, "%d/%m/%Y").ok()?;
Some(DateTime::<Utc>::from_naive_utc_and_offset(
@ -73,6 +52,12 @@ pub fn parse_normalize_phone(inp: String) -> Option<String> {
Some(parsed.to_string())
}
pub fn complete_date(inp: NaiveDate) -> DateTime<Utc> {
DateTime::<Utc>::from_naive_utc_and_offset(
inp.and_hms_opt(0, 0, 0).unwrap(),
Utc
)
}
pub fn normalize_str(subject: String) -> String {
subject.trim().replace("\n", ";").to_string()
@ -108,3 +93,32 @@ pub fn normalize_last_name(subject: String) -> String {
.to_uppercase()
}
/// https://serde.rs/field-attrs.html
pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result<DateTime<Utc>, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
DateTime::parse_from_rfc3339(&s)
.map_err(serde::de::Error::custom)
.map(|dt| dt.with_timezone(&Utc))
}
pub fn deserialize_date<'de, D>(deserializer: D) -> Result<NaiveDate, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
NaiveDate::parse_from_str(&s, "%Y-%m-%d")
.map_err(serde::de::Error::custom)
}
pub fn deserialize_json_list<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
serde_json::from_str(&s).map_err(serde::de::Error::custom)
}

7
sync_csv.sh Executable file
View file

@ -0,0 +1,7 @@
#!/usr/bin/sh
xlsx2csv -n "Adhérents PAPIER" /warmd/etoiledebethleem/copyparty/adhesions/wip_matthieu_b.xlsx \
| sed ':a;N;$!ba;s/\(Champ complémentaire [0-9]\)\n/\1 /g' \
| sed 's/Champ complémentaire \([0-9]\)/CC \1/g' \
| cargo run -- --source csv