Add raw data to database

8 files changed, 89 insertions, 3 deletions

diff --git a/migrations/2020-02-03-225040_tasks/down.sql b/migrations/2020-02-03-225040_tasks/down.sql
new file mode 100644
index 0000000..4827072
--- /dev/null
+++ b/migrations/2020-02-03-225040_tasks/down.sql
@@ -0,0 +1 @@
+drop table tasks;
diff --git a/migrations/2020-02-03-225040_tasks/up.sql b/migrations/2020-02-03-225040_tasks/up.sql
new file mode 100644
index 0000000..faebc0a
--- /dev/null
+++ b/migrations/2020-02-03-225040_tasks/up.sql
@@ -0,0 +1,7 @@
+create table tasks (
+  id bigserial not null primary key,
+  state varchar(8) not null,
+  start_at timestamptz not null,
+  username varchar not null references users(username),
+  payload jsonb not null
+);
diff --git a/migrations/2020-02-04-105747_raw_data/down.sql b/migrations/2020-02-04-105747_raw_data/down.sql
new file mode 100644
index 0000000..fbfd854
--- /dev/null
+++ b/migrations/2020-02-04-105747_raw_data/down.sql
@@ -0,0 +1 @@
+drop table raw_data;
diff --git a/migrations/2020-02-04-105747_raw_data/up.sql b/migrations/2020-02-04-105747_raw_data/up.sql
new file mode 100644
index 0000000..386ba84
--- /dev/null
+++ b/migrations/2020-02-04-105747_raw_data/up.sql
@@ -0,0 +1,8 @@
+create table raw_data(
+  data_type varchar(8) not null,
+  id bigint not null,
+  username varchar not null references users(username),
+  payload jsonb,
+
+  primary key(data_type, id)
+);
diff --git a/src/db.rs b/src/db.rs
index 548ca99..5135b91 100644
--- a/src/db.rs
+++ b/src/db.rs
@@ -155,3 +155,11 @@ pub fn take_task(
         Ok(task)
     })
 }
+
+pub fn insert_data(conn: &PgConnection, data: &models::RawData) -> Result<usize, Error> {
+    use crate::schema::raw_data;
+    let rows = diesel::insert_into(raw_data::table)
+        .values(data)
+        .execute(conn)?;
+    Ok(rows)
+}
diff --git a/src/importer.rs b/src/importer.rs
index 05d56fd..5f5790a 100644
--- a/src/importer.rs
+++ b/src/importer.rs
@@ -140,15 +140,25 @@ fn import_strava_user<S: strava::StravaApi>(
 
         let result = strava.get("/athlete/activities", &token.access_token, &params[..])?;
 
+        let json_error = || Error::UnexpectedJson(result.clone());
         let result = result
             .as_array()
-            .ok_or(Error::UnexpectedJson(result.clone()))?;
+            .ok_or_else(json_error)?;
 
         for activity in result {
+            let id = activity["id"].as_i64().ok_or_else(json_error)?;
             info!(
                 "activity id: {} start: {}",
-                activity["id"], activity["start_date"]
+                id, activity["start_date"]
             );
+
+            db::insert_data(&shared.conn.lock().unwrap(),
+                            &models::RawData {
+                                data_type: models::DataType::StravaActivity,
+                                id: id,
+                                username: username.to_string(),
+                                payload: activity.clone(),
+                            })?;
         }
 
         if result.len() < per_page {
diff --git a/src/models.rs b/src/models.rs
index 3d7eb24..cd8ddf1 100644
--- a/src/models.rs
+++ b/src/models.rs
@@ -1,3 +1,4 @@
+use crate::schema::raw_data;
 use crate::schema::config;
 use crate::schema::strava_tokens;
 use crate::schema::tasks;
@@ -104,3 +105,37 @@ pub struct StravaToken {
     pub access_token: String,
     pub expires_at: DateTime<Utc>,
 }
+
+#[derive(PartialEq, Debug, Clone, Copy, AsExpression, FromSqlRow)]
+#[sql_type = "sql_types::Text"]
+pub enum DataType {
+    StravaActivity = 0,
+}
+
+impl ToSql<sql_types::Text, Pg> for DataType {
+    fn to_sql<W: Write>(&self, out: &mut Output<W, Pg>) -> serialize::Result {
+        let t = match *self {
+            DataType::StravaActivity => "s:activi".to_string(),
+        };
+        <String as ToSql<sql_types::Text, Pg>>::to_sql(&t, out)
+    }
+}
+
+impl FromSql<sql_types::Text, Pg> for DataType {
+    fn from_sql(bytes: Option<&[u8]>) -> deserialize::Result<Self> {
+        let s = <String as FromSql<sql_types::Text, Pg>>::from_sql(bytes)?;
+        match s.as_str() {
+            "s:activi" => Ok(DataType::StravaActivity),
+            &_ => Err("Unrecognized data type".into()),
+        }
+    }
+}
+
+#[derive(Insertable, Queryable)]
+#[table_name = "raw_data"]
+pub struct RawData {
+    pub data_type: DataType,
+    pub id: i64,
+    pub username: String,
+    pub payload: Value,
+}
diff --git a/src/schema.rs b/src/schema.rs
index e605bda..df20c4f 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -8,6 +8,15 @@ table! {
 }
 
 table! {
+    raw_data (data_type, id) {
+        data_type -> Varchar,
+        id -> Int8,
+        username -> Varchar,
+        payload -> Nullable<Jsonb>,
+    }
+}
+
+table! {
     strava_tokens (username) {
         username -> Varchar,
         refresh_token -> Varchar,
@@ -33,7 +42,14 @@ table! {
     }
 }
 
+joinable!(raw_data -> users (username));
 joinable!(strava_tokens -> users (username));
 joinable!(tasks -> users (username));
 
-allow_tables_to_appear_in_same_query!(config, strava_tokens, tasks, users,);
+allow_tables_to_appear_in_same_query!(
+    config,
+    raw_data,
+    strava_tokens,
+    tasks,
+    users,
+);