Output data file renamed, Add detection code to just read file

README.md

@@ -27,6 +27,9 @@ huak run main
 
 # pre-processing data
 huak run preproc
+
+# detect AI-generated data
+huak run detect
 ```
 
 ## Format

pyproject.toml

@@ -23,3 +23,4 @@ dev = [
 [tool.huak.task]
 main = "python3 src/semantic_hashing_demo/main.py"
 preproc = "python3 src/semantic_hashing_demo/preprocessing.py"
+detect = "python3 src/semantic_hashing_demo/detection.py"

src/semantic_hashing_demo/config.py

@@ -12,6 +12,8 @@
 # So, parse accordingly depending on the computational resources for bucketing.
 data_file = "./data/fine_food_reviews_1k.csv"
 
+preprocessed_data_file = "./output/preprocessed_data.csv"
+
 # no. of text samples
 n = 20
 

src/semantic_hashing_demo/detection.py

@@ -0,0 +1,24 @@
+""" 
+The end goal is to detect AI-generated text.
+"""
+import polars as pl
+from config import preprocessed_data_file
+
+
+def main():
+    # Specify data types for multiple columns to be read as strings
+    dtype_spec = {
+        "Hash 8-bit": str,
+        "Hash 16-bit": str,
+        "Hash 32-bit": str,
+        "Hash 64-bit": str,
+        "Hash 128-bit": str,
+    }
+
+    # pull data into polars dataframe
+    df = pl.read_csv(preprocessed_data_file, dtypes=dtype_spec)
+    print(df.head())
+
+
+if __name__ == "__main__":
+    main()