From c1a9a169b1d1da4861871c90b2416ea05d975152 Mon Sep 17 00:00:00 2001 From: Ben Artuso Date: Fri, 18 Aug 2023 12:07:23 -0400 Subject: [PATCH] redis docker - draft PR --- Dockerfile | 1 + docker-compose.yml | 5 +++ src/lib/src/api/remote/commits.rs | 2 +- src/lib/src/core/index/pusher.rs | 6 ++-- src/server/src/app_data.rs | 3 +- src/server/src/controllers/commits.rs | 42 +++++----------------- src/server/src/main.rs | 15 ++++---- src/server/src/tasks/post_push_complete.rs | 3 +- src/server/src/test.rs | 6 ---- 9 files changed, 28 insertions(+), 55 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1e99911fd..b24e70cd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,5 +49,6 @@ FROM debian:bullseye-slim AS runtime WORKDIR /oxen-server COPY --from=builder /usr/src/oxen-server/target/release/oxen-server /usr/local/bin ENV SYNC_DIR=/var/oxen/data +ENV REDIS_URL=redis://redis/ EXPOSE 3001 CMD ["oxen-server", "start", "-p", "3001"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 821c882f1..f3a58c6cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,3 +17,8 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.oxen.rule=Host(`0.0.0.0`)" + redis: + image: redis:latest + ports: + - "6379:6379" + diff --git a/src/lib/src/api/remote/commits.rs b/src/lib/src/api/remote/commits.rs index cfd7c01ee..efddac521 100644 --- a/src/lib/src/api/remote/commits.rs +++ b/src/lib/src/api/remote/commits.rs @@ -388,7 +388,7 @@ pub async fn bulk_post_push_complete( ) -> Result<(), OxenError> { use serde_json::json; - let uri = format!("/commits/complete"); + let uri = "/commits/complete".to_string(); let url = api::endpoint::url_from_repo(remote_repo, &uri)?; log::debug!("bulk_post_push_complete: {}", url); let body = serde_json::to_string(&json!(commits)).unwrap(); diff --git a/src/lib/src/core/index/pusher.rs b/src/lib/src/core/index/pusher.rs index af33f15db..86e167997 100644 --- a/src/lib/src/core/index/pusher.rs +++ b/src/lib/src/core/index/pusher.rs @@ -381,7 +381,7 @@ async fn push_missing_commit_dbs( async fn push_missing_commit_entries( local_repo: &LocalRepository, remote_repo: &RemoteRepository, - branch: &Branch, + _branch: &Branch, commits: &Vec, ) -> Result<(), OxenError> { log::debug!("rpush_entries num unsynced {}", commits.len()); @@ -418,13 +418,13 @@ async fn push_missing_commit_entries( // Treat this as one giant commit for testing - the commit dbs will know the difference. - if unsynced_entries.len() != 0 { + if !unsynced_entries.is_empty() { let all_entries = UnsyncedCommitEntries { commit: commits[0].to_owned(), entries: unsynced_entries, }; - let bar = Arc::new(ProgressBar::new(total_size as u64)); + let bar = Arc::new(ProgressBar::new(total_size)); push_entries( local_repo, diff --git a/src/server/src/app_data.rs b/src/server/src/app_data.rs index 63959f91f..da7a7743a 100644 --- a/src/server/src/app_data.rs +++ b/src/server/src/app_data.rs @@ -1,5 +1,4 @@ -use redis::Client; -use std::{path::PathBuf, sync::Arc}; +use std::path::PathBuf; #[derive(Debug, Clone)] pub struct OxenAppData { diff --git a/src/server/src/controllers/commits.rs b/src/server/src/controllers/commits.rs index 4948e8a56..1fc02a6f2 100644 --- a/src/server/src/controllers/commits.rs +++ b/src/server/src/controllers/commits.rs @@ -179,7 +179,7 @@ pub async fn latest_synced(req: HttpRequest) -> actix_web::Result { match content_validator::is_valid(&repository, &commit) { Ok(true) => { @@ -203,15 +203,15 @@ pub async fn latest_synced(req: HttpRequest) -> actix_web::Result { // Desired behavior here? log::error!("latest_synced content_validator::is_valid error {err:?}"); - return Ok( - HttpResponse::InternalServerError().json(IsValidStatusMessage { + return Ok(HttpResponse::InternalServerError().json( + IsValidStatusMessage { status: String::from(STATUS_ERROR), status_message: String::from(MSG_INTERNAL_SERVER_ERROR), status_description: format!("Err: {err:?}"), is_processing: false, is_valid: false, - }), - ); + }, + )); } } } @@ -891,7 +891,7 @@ pub async fn complete_bulk(req: HttpRequest, body: String) -> Result with serde let commits: Vec = match serde_json::from_str(&body) { Ok(commits) => commits, @@ -900,8 +900,8 @@ pub async fn complete_bulk(req: HttpRequest, body: String) -> Result Result Result { - // log::debug!( - // "Success processing commit {:?} on repo {:?}", - // commit, - // &repo_path_clone - // ); - // } - // Err(err) => { - // log::error!( - // "Could not process commit {:?} on repo {:?}: {}", - // commit, - // &repo_path_clone, - // err - // ); - // } - // }; - log::debug!("continuing on with the controller for now...") } Ok(HttpResponse::Ok().json(StatusMessage::resource_created())) } diff --git a/src/server/src/main.rs b/src/server/src/main.rs index 58dee4155..797ea31d1 100644 --- a/src/server/src/main.rs +++ b/src/server/src/main.rs @@ -1,7 +1,5 @@ -use actix_http::Request; -use actix_web::web::{Bytes, BytesMut}; use liboxen::config::UserConfig; -use liboxen::error::OxenError; + use liboxen::model::User; pub mod app_data; @@ -18,18 +16,17 @@ pub mod view; extern crate log; use actix_web::middleware::{Condition, Logger}; -use actix_web::{web, App, HttpRequest, HttpResponse, HttpServer}; +use actix_web::{web, App, HttpServer}; use actix_web_httpauth::middleware::HttpAuthentication; -use bincode; + use clap::{Arg, Command}; use env_logger::Env; -use redis; + use serde::{Deserialize, Serialize}; use std::path::Path; use std::time::Duration; use tokio::time::sleep; -use crate::errors::OxenHttpError; use crate::tasks::post_push_complete::PostPushComplete; const VERSION: &str = liboxen::constants::OXEN_VERSION; @@ -58,7 +55,9 @@ async fn main() -> std::io::Result<()> { // Redis polling worker setup async fn run_redis_poller() { - let redis_client = redis::Client::open("redis://127.0.0.1/").unwrap(); + let redis_url = + std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost/".to_string()); + let redis_client = redis::Client::open(redis_url).expect("Failed to connect to redis"); let mut con = redis_client.get_connection().unwrap(); loop { diff --git a/src/server/src/tasks/post_push_complete.rs b/src/server/src/tasks/post_push_complete.rs index 96e24bff5..b4eb2f7b6 100644 --- a/src/server/src/tasks/post_push_complete.rs +++ b/src/server/src/tasks/post_push_complete.rs @@ -3,7 +3,6 @@ use liboxen::{ model::{Commit, LocalRepository}, }; use serde::{Deserialize, Serialize}; -use std::{thread, time}; #[derive(Serialize, Deserialize, Debug)] pub struct PostPushComplete { @@ -12,7 +11,7 @@ pub struct PostPushComplete { } impl PostPushComplete { - pub fn run(self) -> () { + pub fn run(self) { log::debug!( "Running cachers for commit {:?} on repo {:?} from redis queue", self.commit.id, diff --git a/src/server/src/test.rs b/src/server/src/test.rs index 2ab1bcba8..a75aa0a9e 100644 --- a/src/server/src/test.rs +++ b/src/server/src/test.rs @@ -25,12 +25,6 @@ pub fn get_sync_dir() -> Result { std::fs::create_dir_all(&sync_dir)?; Ok(sync_dir) } - -pub fn get_redis_client() -> Result, OxenError> { - let redis_url = std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost".to_string()); - let client = redis::Client::open(redis_url.as_str())?; - Ok(Arc::new(client)) -} pub fn create_local_repo( sync_dir: &Path, namespace: &str,