From e9a74a0c655237270fccf67afe69129871a08385 Mon Sep 17 00:00:00 2001 From: Donovan Mueller Date: Wed, 23 Feb 2022 10:50:33 -0700 Subject: [PATCH] wip: update config schema/examples for file_stores Specifically, Config.Rdb_dest.file_stores. This code is untested but compiles up to the point that something outside the config tries to access the old files_path field. issue #4 --- _env/config.example.json | 19 +++++++++-- app/bin/config/dune | 2 +- app/bin/config/file_store.atd | 9 ++++++ app/bin/config/rdb_dest.atd | 3 +- app/bin/config/rdb_dest.ml | 60 +++++++++++++++++++++++++++++++++++ docker-compose.dev.yml | 6 ++-- docker-compose.yml | 6 +++- 7 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 app/bin/config/file_store.atd create mode 100644 app/bin/config/rdb_dest.ml diff --git a/_env/config.example.json b/_env/config.example.json index 63dec38..47f8c0f 100644 --- a/_env/config.example.json +++ b/_env/config.example.json @@ -32,9 +32,22 @@ , "log_namespace" : "Source" } , "rdb_dest" : - { // Absolute path to where the video and thumbnail files should be - // downloaded to within the Docker container. - "files_path" : "/data/files" + { // Where to store files. Multiple locations can be specified to accomodate + // spanning multiple volumes. The system distributes files based on order + // of location definition (first to last), available space, and max allowed + // usage. + // + // Each location is defined by a "path" and "max_usage". Path should be + // absolute. Max usage follows the PHP-style suffixes of K, M, G, and T + // (kilo-, mega-, giga-, and terabytes respectively). Case-insensitive. If + // no suffix is given, then the number will be taken as bytes. Setting + // "max_usage" to `null` means use as much storage as is available. + // + // See https://www.php.net/manual/en/faq.using.php#faq.using.shorthandbytes + "file_stores" : + [ {"path": "/data/files/volume_a", "max_usage": "3G"} + , {"path": "/data/files/volume_b", "max_usage": "4G"} + , {"path": "/data/files/volume_c", "max_usage": null} ] , "log_level" : "info" , "log_namespace" : "Dest" } diff --git a/app/bin/config/dune b/app/bin/config/dune index a6cf9ea..72ee5b9 100644 --- a/app/bin/config/dune +++ b/app/bin/config/dune @@ -1,3 +1,3 @@ (library (name config) - (libraries atdgen batteries)) + (libraries atdgen batteries re re.perl)) diff --git a/app/bin/config/file_store.atd b/app/bin/config/file_store.atd new file mode 100644 index 0000000..bdea91b --- /dev/null +++ b/app/bin/config/file_store.atd @@ -0,0 +1,9 @@ + + +type max_usage = string option wrap + + +type t = + { path : string + ; max_usage : max_usage + } \ No newline at end of file diff --git a/app/bin/config/rdb_dest.atd b/app/bin/config/rdb_dest.atd index bbb99c2..c9c77c3 100644 --- a/app/bin/config/rdb_dest.atd +++ b/app/bin/config/rdb_dest.atd @@ -1,7 +1,8 @@ type log_level = abstract +type file_store = abstract type t = - { files_path : string + { file_stores : file_store list ; log_level : log_level ; log_namespace : string } \ No newline at end of file diff --git a/app/bin/config/rdb_dest.ml b/app/bin/config/rdb_dest.ml new file mode 100644 index 0000000..c9fef3a --- /dev/null +++ b/app/bin/config/rdb_dest.ml @@ -0,0 +1,60 @@ + + +module File_store = struct + module Max_usage = struct + type t = int option + + let valid_pattern = "^([1-9]\\d*)([kmgt])?$" |> Re.Perl.compile_pat + + let clean s = + let separators = "[_, ]+" |> Re.Perl.compile_pat in + s + |> Re.replace_string separators ~by:"" + |> String.lowercase_ascii + + let validate s = + match s with + | None -> true + | Some s -> + s |> clean |> Re.execp valid_pattern + + let wrap s = + if not (validate s) then failwith "Invalid byte string." + else + + match s with + | None -> + None + + | Some s -> begin + let matches = s |> clean |> Re.exec valid_pattern in + match matches |> Re.Group.all with + | [| _; amount; "" |] -> + Some (amount |> int_of_string) + + | [| _; amount; suffix |] -> begin + let amount = amount |> int_of_string in + match suffix with + | "k" -> Some (amount * 1024) + | "m" -> Some (amount * 1024 * 1024) + | "g" -> Some (amount * 1024 * 1024 * 1024) + | "t" -> Some (amount * 1024 * 1024 * 1024 * 1024) + | suffix -> + Printf.sprintf "Unknown suffix '%s'" suffix + |> failwith + end + + | _ -> + failwith "Unexpected array structure returned from Re.Group.all" + end + + let unwrap t = + match t with + | None -> + None + + | Some b -> + Some (b |> string_of_int) + + end +end \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ce3d6a9..7d4ed19 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -11,8 +11,10 @@ services: # Required for building. - ./app:${APP_PREFIX_DIR}/app - # Required for running app - # - ./_env/config.json:${APP_PREFIX_DIR}/_env/config.json:ro + # File stores + - ./_data/app/store_a:${APP_PREFIX_DIR}/_data/store_a + - ./_data/app/store_b:${APP_PREFIX_DIR}/_data/store_b + - /Volumes/WILLY/OVPSync:${APP_PREFIX_DIR}/_data/store_c db: ports: [ "3306:3306" ] diff --git a/docker-compose.yml b/docker-compose.yml index 71fc0e9..9efbfe5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,11 @@ services: volumes: - ./app/entrypoint.sh:${APP_PREFIX_DIR}/entrypoint.sh - ./_env/config.json:${APP_PREFIX_DIR}/_env/config.json:ro - - ./_data/app:${APP_PREFIX_DIR}/_data + # File store volumes should be defined in docker-compose.override.yml. + # They need to match up with the [rdb_dest.file_store] parameter in + # _env/config.json. Below values are just examples. + # - ./_data/app:${APP_PREFIX_DIR}/_data/volume_a + # - /Volumes/OVPSync_extra:${APP_PREFIX_DIR}/_data/volume_b depends_on: - db