Compare commits

..

No commits in common. "853a3be680225141522110ba5a99057270102b6d" and "a604d63622da149b68db4134a76eb0ab065bb037" have entirely different histories.

7 changed files with 365 additions and 890 deletions

663
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -7,7 +7,7 @@ edition = "2021"
clap = { version = "4.0.32", features = ["derive"] }
serde = { version = "1.0", features = ["derive"]}
serde_json = "1.0"
reqwest = { version = "0.11", features = ["json", "multipart", "stream"] }
reqwest = { version = "0.11", features = ["json"] }
tokio = { version = "1", features = ["full"] }
url = "2.4.1"
xdg = "2.5.2"
@ -22,4 +22,3 @@ rand = "0.8.5"
phonenumber = "0.3.3"
email_address = "0.2"
fully_pub = "0.1.4"
base64_light = "0.1.4"

View file

@ -17,30 +17,3 @@ Written in Rust.
Create a `.env` file from `.env.example` and fill in some secrets.
Run the program `dotenv cargo run`
## fonctionnements
- On va déjà récupérer la liste des utilisateurs en mode "summary" UserSummary` cad avec juste l'id, l'email, le prénom et le nom.
- Ensuite on va récupérer la liste des transactions avec helloasso pour cette période comptable
- ce qui compte c'est d'avoir la référence comptable
- on peut faire une req sql qui filtre sur les id ou alors sur un compte
- using accounting/years/{ID_YEAR}/account/journal (GET)
- ou alors req SQL
- On va créer une liste de réponse helloasso à traiter en filtrant les "réponses" d'helloasso. Pour chaque réponse, si l'id helloasso de cette réponse est trouvé dans la liste récupéré avant sur Paheko, alors on l'ignore. Sinon on garde
- Pour chaque réponse à traiter
- On va regarder si l'id de la réponse est trouvé dans une écriture comptable, si oui on ignore
- cela veut dire que la personne est déjà inscrite pour ajd
- On va regarder si l'email est déjà présent dans la liste des adhérents ou ancien adhérents (à N-1)
- Si non, on créé l'utilisateur
- Si oui, on ne créé pas d'utilisateur, on récupère l'id
- On va créer une activité pour cet utilisateur
- On va créer une écriture comptable, le numéro de la pièce comptable étant "HA/{ID_ANSWER}" (à la base je voulais le faire en JSON pour plus de détails, mais c'est trop gros)
PB: l'API permet d'importer des activités, mais pas de lier une écriture comptable
ya une table `acc_transactions_users` qui permet de lier une transaction avec une activité d'un utilisateur
### références helloasso
le `order.id` et le `answer.id` que retourne l'API d'helloasso sont en fait les mêmes,

View file

@ -2,8 +2,6 @@ use anyhow::{Context, Result, anyhow};
use url::Url;
use serde::{Serialize, Deserialize};
use fully_pub::fully_pub;
use chrono::prelude::{DateTime, Utc};
use crate::utils::deserialize_datetime;
use thiserror::Error;
@ -288,8 +286,9 @@ enum MembershipMode {
#[fully_pub]
struct OrderDetails {
id: u64,
#[serde(deserialize_with = "deserialize_datetime", rename="date")]
inception_time: DateTime<Utc>
// #[serde(with = "date_format")]
// date: DateTime<Utc>
date: String
}
#[derive(Debug, Serialize, Deserialize)]

View file

@ -1,4 +1,3 @@
mod utils;
mod paheko;
mod helloasso;
@ -8,19 +7,34 @@ use chrono::prelude::{NaiveDate, DateTime, Utc, Datelike};
use strum::Display;
use serde::{Serialize, Deserialize};
use std::collections::HashSet;
use rand::{thread_rng, Rng};
use phonenumber;
use utils::generate_id;
use paheko::UserSummary;
/// ID
#[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq, Hash)]
pub struct Id(pub u64);
impl Id {
pub fn to_string(&self) -> String {
format!("{:x}", self.0)
}
}
impl Into<String> for Id {
fn into(self) -> String {
format!("{:x}", self.0)
}
}
pub fn generate_id() -> Id {
Id(thread_rng().gen())
}
/// permanent config to store long-term config
/// used to ingest env settings
#[derive(Deserialize, Serialize, Debug)]
struct Config {
helloasso_email: String,
helloasso_password: String,
paheko_base_url: String,
paheko_client_id: String,
paheko_client_secret: String,
helloasso_password: String
}
static APP_USER_AGENT: &str = concat!(
@ -128,6 +142,48 @@ async fn get_auth_client_from_cache(
}
#[derive(Debug, Serialize, Deserialize, Clone)]
struct HelloassoReferences {
answer_id: u64,
order_id: u64
// payment_id: u64,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
struct ExternalReferences {
helloasso_ref: HelloassoReferences
}
/// for now we include the custom fields into the paheko user
/// we don't have time to implement user settings to change the custom fields mapping
/// for now, manual mapping
#[derive(Debug, Serialize, Deserialize, Clone)]
struct PahekoUser {
id: Id,
first_name: String,
last_name: String,
email: Option<String>,
phone: Option<String>,
address: String,
city: String,
postal_code: String,
country: String,
skills: Option<String>,
job: Option<String>,
birthday: Option<u32>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
struct PahekoMembership {
id: Id,
users: Vec<Id>,
campaign: String,
mode: helloasso::MembershipMode,
inception_datum: DateTime<Utc>,
external_references: ExternalReferences
}
/// rust how to access inner enum value
#[derive(Debug, PartialEq, Clone, Copy)]
enum HelloassoCustomFieldType {
@ -184,10 +240,6 @@ fn parse_normalize_phone(phone_number_opt: Option<String>) -> Option<String> {
}
fn normalize_str(subject: String) -> String {
subject.trim().replace("\n", ";").to_string()
}
/// remove year precision to comply with GDPR eu rules
fn parse_and_get_birthday_year(raw_date: String) -> Option<u32> {
let d_res = NaiveDate::parse_from_str(&raw_date.trim(), "%d/%m/%Y");
@ -195,31 +247,12 @@ fn parse_and_get_birthday_year(raw_date: String) -> Option<u32> {
Some(d.year().try_into().ok()?)
}
fn helloasso_to_paheko_membership(helloasso_membership: &helloasso::MembershipMode) -> paheko::MembershipMode {
match helloasso_membership {
helloasso::MembershipMode::Couple => paheko::MembershipMode::Couple,
helloasso::MembershipMode::Individual => paheko::MembershipMode::Individual,
helloasso::MembershipMode::BenefactorCouple => paheko::MembershipMode::BenefactorCouple,
helloasso::MembershipMode::BenefactorIndividual => paheko::MembershipMode::BenefactorIndividual
}
}
async fn launch_adapter() -> Result<()> {
dotenvy::dotenv()?;
let config: Config = envy::from_env().context("Failed to load env vars")?;
let mut user_cache = load_user_cache().context("Failed to load user cache")?;
let mut paheko_client: paheko::Client = paheko::Client::new("http://localhost:8082/api/".to_string());
let paheko_credentials = paheko::Credentials {
client_id: config.paheko_client_id,
client_secret: config.paheko_client_secret
};
let paheko_client: paheko::AuthentifiedClient = paheko_client.login(paheko_credentials).await?;
let mut ha_client: helloasso::Client = Default::default();
let login_payload = helloasso::LoginPayload {
@ -243,12 +276,16 @@ async fn launch_adapter() -> Result<()> {
// get the list of payments associated
// first step: output a list of PahekoUser with PahekoMembership
let pk_memberships: Vec<paheko::Membership> = vec![];
let mut pk_users: Vec<paheko::User> = vec![];
let mut pk_memberships: Vec<paheko::Membership> = vec![];
let pk_memberships: Vec<PahekoMembership> = vec![];
let mut pk_users: Vec<PahekoUser> = vec![];
let mut pk_memberships: Vec<PahekoMembership> = vec![];
let mut count: u64 = 0;
let mut names: HashSet<String> = HashSet::new();
// read_custom_field(&answer, HelloAssoCustomFieldType::Email).or(Some(answer.payer_user.email.clone())),
use email_address::*;
use std::str::FromStr;
fn choose_email(answer: &helloasso::FormAnswer) -> Option<String> {
read_custom_field(&answer, HelloassoCustomFieldType::Email)
.and_then(|x| {
@ -261,71 +298,37 @@ async fn launch_adapter() -> Result<()> {
.or(Some(answer.payer_user.email.clone()))
}
// get summary of users
let existing_users = paheko_client.get_users().await.context("Get users")?;
// get summary of transactions for that year
let existing_transactions = paheko_client.get_transactions(1).await.context("Get transactions")?;
// TODO: before creating any users, get the current maximum id of the users table to predict
// the next auto-incrementing id.
for answer in answers {
eprintln!("Processing answer:");
let email = choose_email(&answer);
eprintln!(" email: {:?}", email);
let paheko_user = paheko::User {
// TODO: parse birthday
dbg!(&answer);
for custom_field in answer.custom_fields.iter() {
names.insert(custom_field.name.clone());
count += 1;
}
let paheko_user = PahekoUser {
id: generate_id(),
first_name: Some(normalize_str(answer.user.first_name.clone())),
last_name: normalize_str(answer.user.last_name.clone()),
email,
first_name: answer.user.first_name.clone(),
last_name: answer.user.last_name.clone(),
email: choose_email(&answer),
phone: parse_normalize_phone(read_custom_field(&answer, HelloassoCustomFieldType::Phone)),
skills: read_custom_field(&answer, HelloassoCustomFieldType::Skills).map(normalize_str),
address: read_custom_field(&answer, HelloassoCustomFieldType::Address)
.map(normalize_str)
.expect("to have address"),
skills: read_custom_field(&answer, HelloassoCustomFieldType::Skills),
address: read_custom_field(&answer, HelloassoCustomFieldType::Address).expect("to have address"),
postal_code: read_custom_field(&answer, HelloassoCustomFieldType::PostalCode).expect("to have postal code"),
city: read_custom_field(&answer, HelloassoCustomFieldType::City)
.map(normalize_str)
.expect("to have city"),
country: answer.payer_user.country.clone().trim()[..=1].to_string(), // ISO 3166-1 alpha-2
job: read_custom_field(&answer, HelloassoCustomFieldType::Job).map(normalize_str),
birth_year: read_custom_field(&answer, HelloassoCustomFieldType::Birthday).and_then(parse_and_get_birthday_year),
register_time: answer.order.inception_time,
city: read_custom_field(&answer, HelloassoCustomFieldType::City).expect("to have city"),
country: answer.payer_user.country.clone(),
job: read_custom_field(&answer, HelloassoCustomFieldType::Job),
birthday: read_custom_field(&answer, HelloassoCustomFieldType::Birthday).and_then(parse_and_get_birthday_year),
// FIXME: the reference will be in the data of the paheko activity, and will only
// reference the answer id
};
// check for existing transactions
if let Some(_) = existing_transactions.iter().find(
|summary| summary.reference == format!("HA/{}", answer.id)
) {
eprintln!(" Skipped: existing transaction found");
continue;
}
// check for existing paheko user, or create paheko user
let paheko_user_summary = match existing_users.iter().find(|user| user.email == paheko_user.email) {
Some(user) => user.clone(),
None => {
let c = paheko_client.create_user(&paheko_user).await.context("Expected to create paheko user")?;
eprintln!(" Created paheko user");
UserSummary {
id: utils::Id(0),
first_name: paheko_user.first_name.clone(),
last_name: paheko_user.last_name.clone(),
email: paheko_user.email.clone()
}
}
};
let mut pk_membership = paheko::Membership {
let mut pk_membership = PahekoMembership {
id: generate_id(),
campaign: "".to_string(),
inception_time: Utc::now(),
mode: helloasso_to_paheko_membership(&answer.mode),
inception_datum: Utc::now(),
mode: answer.mode.clone(),
users: vec![paheko_user.id.clone()],
external_references: paheko::ExternalReferences {
helloasso_ref: paheko::HelloassoReferences {
external_references: ExternalReferences {
helloasso_ref: HelloassoReferences {
answer_id: answer.id,
order_id: answer.order.id
}
@ -341,16 +344,16 @@ async fn launch_adapter() -> Result<()> {
second_pk_user.phone = None;
second_pk_user.skills = None;
second_pk_user.job = None;
second_pk_user.birth_year = None;
second_pk_user.birthday = None;
// add first_name
match read_custom_field(&answer, HelloassoCustomFieldType::LinkedUserFirstName) {
Some(name) => {
second_pk_user.first_name = Some(name);
second_pk_user.first_name = name;
},
None => {
second_pk_user.first_name = None;
eprintln!("Warn: Got a user with Couple mode but no additional name given!")
second_pk_user.first_name = "Conjoint".to_string();
eprintln!("Got a user with Couple mode but no additional name given!")
}
}
@ -365,6 +368,30 @@ async fn launch_adapter() -> Result<()> {
dbg!(&pk_users.len());
dbg!(&pk_memberships.len());
// println!("{:?}", &pk_users.iter().map(|user| format!("{:?}", user.email)).collect::<Vec<String>>());
for u in pk_users.iter() {
println!("{:?}", (&u.first_name, &u.last_name, &u.email, &u.phone, &u.birthday, &u.country));
}
for u in pk_users.iter() {
let email = u.email.clone();
if email.is_none() { continue; }
println!("{:?},{:?}", email.unwrap(), format!("{} {}", &u.first_name, &u.last_name));
}
// then, request the current list of users
// match with the email address
// we consider the email address as the id for a helloasso user
// then, upload the PahekoMembership
// in paheko, there is a custom field "external extensions data" which can be used to put an
// id,
// for each uses we extracted
// we check if there is an existing user by checking for the ha forn answer id
Ok(())
}

View file

@ -1,304 +0,0 @@
use anyhow::{Context, Result, anyhow};
use url::Url;
use serde::{Serialize, Deserialize};
use fully_pub::fully_pub;
use crate::utils::Id;
use chrono::prelude::{NaiveDate, DateTime, Utc, Datelike};
use thiserror::Error;
#[derive(Debug, Serialize, Deserialize, Clone)]
#[fully_pub]
struct HelloassoReferences {
answer_id: u64,
order_id: u64
// payment_id: u64,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[fully_pub]
struct ExternalReferences {
helloasso_ref: HelloassoReferences
}
/// for now we include the custom fields into the paheko user
/// we don't have time to implement user settings to change the custom fields mapping
/// for now, manual mapping
#[derive(Debug, Serialize, Deserialize, Clone)]
#[fully_pub]
struct User {
id: Id,
first_name: Option<String>,
last_name: String,
email: Option<String>,
phone: Option<String>,
address: String,
city: String,
postal_code: String,
country: String,
skills: Option<String>,
job: Option<String>,
birth_year: Option<u32>,
register_time: DateTime<Utc>
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[fully_pub]
struct UserSummary {
id: Id,
first_name: Option<String>,
last_name: String,
email: Option<String>
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
#[fully_pub]
enum MembershipMode {
Individual,
Couple,
BenefactorIndividual,
BenefactorCouple,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[fully_pub]
struct Membership {
id: Id,
users: Vec<Id>,
campaign: String,
mode: MembershipMode,
inception_time: DateTime<Utc>,
external_references: ExternalReferences
}
#[derive(Debug)]
#[fully_pub]
struct Client {
client: reqwest::Client,
base_url: Url,
}
#[derive(Error, Debug)]
enum APIClientError {
#[error("Received non-normal status code from API")]
InvalidStatusCode
}
#[derive(Debug, Clone)]
#[fully_pub]
struct Credentials {
client_id: String,
client_secret: String
}
impl Default for Client {
fn default() -> Self {
Client {
client: Client::get_base_client_builder()
.build()
.expect("Expected reqwest client to be built"),
base_url: Url::parse("https://paheko.etoiledebethleem.fr/api/") // the traling slash is important
.expect("Expected valid paheko API base URL")
}
}
}
use base64_light::base64_encode;
fn build_auth_headers(credentials: &Credentials) -> reqwest::header::HeaderMap {
let mut login_headers = reqwest::header::HeaderMap::new();
login_headers.insert(
"Authorization",
format!("Basic {}", &base64_encode(
&format!("{}:{}", &credentials.client_id, &credentials.client_secret)
)).parse().expect("Header value to be OK")
);
login_headers
}
impl Client {
pub fn new(base_url: String) -> Client {
Client {
client: Client::get_base_client_builder()
.build()
.expect("Expected reqwest client to be built"),
base_url: Url::parse(&base_url) // the traling slash is important
.expect("Expected valid paheko API base URL")
}
}
fn get_base_client_builder() -> reqwest::ClientBuilder {
let mut default_headers = reqwest::header::HeaderMap::new();
default_headers.insert("Accept", "application/json".parse().unwrap());
let proxy = reqwest::Proxy::http("http://localhost:8998").unwrap();
reqwest::Client::builder()
.proxy(proxy)
.default_headers(default_headers)
}
pub async fn login(&mut self, credentials: Credentials) -> Result<AuthentifiedClient> {
let hypothetic_client = self.authentified_client(credentials);
let query: String = r#"
SELECT key,value FROM config WHERE key="org_name"
"#.to_string();
match hypothetic_client.sql_query(query).await {
Ok(_value) => {
Ok(hypothetic_client)
},
Err(err) => {
Err(anyhow!("Failed to authenticate: Credentials provided are invalids, {:?}", err))
}
}
}
pub fn authentified_client(&self, credentials: Credentials) -> AuthentifiedClient {
AuthentifiedClient::new(self.base_url.clone(), credentials)
}
}
#[derive(Debug, Clone)]
pub struct AuthentifiedClient {
credentials: Credentials,
client: reqwest::Client,
base_url: Url
}
// SELECT id,nom AS first_name,last_name,email,external_custom_data FROM users LIMIT 5;
#[derive(Debug, Deserialize)]
#[fully_pub]
struct SimpleUser {
id: u32,
first_name: String,
last_name: String,
email: Option<String>,
external_custom_data: Option<String>
}
#[derive(Debug, Deserialize)]
#[fully_pub]
struct SqlQueryOutput {
count: u64,
results: serde_json::Value
}
#[derive(Debug, Deserialize)]
#[fully_pub]
struct TransactionSummary {
id: u64,
reference: String
}
impl AuthentifiedClient {
pub fn new(base_url: Url, credentials: Credentials) -> Self {
AuthentifiedClient {
client: Client::get_base_client_builder()
.default_headers(build_auth_headers(&credentials))
.build()
.expect("Expect client to be built"),
credentials,
base_url
}
}
pub async fn sql_query(&self, query: String) -> Result<SqlQueryOutput> {
#[derive(Serialize)]
struct Payload {
sql: String
}
let payload = Payload { sql: query };
let path = self.base_url.join("sql")?;
let res = self.client
.post(path)
.json(&payload)
.send().await?;
if res.status() != 200 {
dbg!(res);
return Err(APIClientError::InvalidStatusCode.into());
}
Ok(res.json().await.context("Sql query")?)
}
pub async fn get_users(&self) -> Result<Vec<UserSummary>> {
let query: String = r#"
SELECT id,nom AS first_name,last_name,email FROM users;
"#.to_string();
let users_val = self.sql_query(query).await.context("Fetching users")?;
Ok(serde_json::from_value(users_val.results)?)
}
pub async fn get_transactions(&self, id_year: u32)
-> Result<Vec<TransactionSummary>>
{
let query: String = format!(r#"
SELECT id,reference FROM acc_transactions WHERE id_year={} AND reference LIKE 'HA/%';
"#, id_year).to_string();
let val = self.sql_query(query).await.context("Fetching transactions")?;
Ok(serde_json::from_value(val.results)?)
}
pub async fn create_user(&self, user: &User)
-> Result<UserSummary>
{
// single-user import
// create virtual file
let u = user.clone();
let mut csv_content: String = String::new();
csv_content.push_str("numero,nom,last_name,adresse,code_postal,ville,pays,telephone,email,annee_naissance,profession,interets,lettre_infos,date_inscription\n");
csv_content.push_str(
format!("{},{:?},{:?},{:?},{:?},{:?},{:?},{:?},{:?},{},{:?},{:?},{},{}\n",
"".to_string(),
u.first_name.clone().unwrap_or("".to_string()),
u.last_name.clone(),
u.address,
u.postal_code,
u.city,
u.country,
u.phone.unwrap_or("".to_string()),
u.email.clone().unwrap_or("".to_string()),
u.birth_year.map(|x| format!("{}", x)).unwrap_or("".to_string()),
u.job.unwrap_or("".to_string()),
u.skills.unwrap_or("".to_string()),
1,
user.register_time.format("%d/%m/%Y")
).as_str());
use reqwest::multipart::Form;
use reqwest::multipart::Part;
let part = Part::text(csv_content).file_name("file");
let form = Form::new()
.part("file", part);
let res = self.client
.post(self.base_url.join("user/import/")?)
.multipart(form)
.send().await?;
if res.status() != 200 {
return Err(APIClientError::InvalidStatusCode.into());
}
Ok(
UserSummary {
id: Id(0),
first_name: u.first_name,
last_name: u.last_name,
email: u.email
}
)
}
}

View file

@ -1,32 +0,0 @@
use serde::{Serialize, Deserialize, Deserializer};
use rand::{thread_rng, Rng};
use chrono::prelude::{DateTime, Utc};
/// ID
#[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq, Hash)]
pub struct Id(pub u64);
impl Id {
pub fn to_string(&self) -> String {
format!("{:x}", self.0)
}
}
impl Into<String> for Id {
fn into(self) -> String {
format!("{:x}", self.0)
}
}
pub fn generate_id() -> Id {
Id(thread_rng().gen())
}
pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result<DateTime<Utc>, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
DateTime::parse_from_rfc3339(&s)
.map_err(serde::de::Error::custom)
.map(|dt| dt.with_timezone(&Utc))
}