add

Jasonbroker · Jan 17, 2017 · 965b1a4 · 965b1a4
1 parent bf63a5a
commit 965b1a4
Show file tree

Hide file tree

Showing 7 changed files with 361 additions and 185 deletions.
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/out/production/crawler/com/company/GithubRepoPageProcessor.class b/out/production/crawler/com/company/GithubRepoPageProcessor.class
diff --git a/src/com/company/FileHelper.java b/src/com/company/FileHelper.java
@@ -1,9 +1,6 @@
 package com.company;
 
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
+import java.io.*;
 import java.util.Properties;
 
 import static java.io.File.separator;
@@ -13,6 +10,7 @@
  */
 public class FileHelper {
 
+    static PrintWriter printWriter;
     public static String jarPath() {
         return Main.class.getProtectionDomain().getCodeSource().getLocation().getPath();
 
@@ -35,6 +33,14 @@ public static String desktopPathWithFileName(String filename) {
         return  file.getAbsolutePath() + separator + "Desktop"+ separator + filename;
     }
 
+    public static void writeToConfigFile(String string) throws IOException {
+        if (printWriter == null) {
+            printWriter = new PrintWriter(new FileWriter(pathInCodeFolder("config.txt"), false));
+        }
+        printWriter.print(string);
+        printWriter.close();
+    }
+
     public static String readFileByLines(String fileName) throws IOException {
         File file = new File(fileName);
         BufferedReader reader = null;

diff --git a/src/com/company/GithubRepoPageProcessor.java b/src/com/company/GithubRepoPageProcessor.java
@@ -19,6 +19,10 @@ class GithubRepoPageProcessor implements PageProcessor {
 
     private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setTimeOut(60000);
     private String regString = null;
+
+    private boolean backspaceHead;
+    private boolean backspaceTail;
+
     public boolean strict = false;
     public String[] operators;
     public int depth = 0;
@@ -27,6 +31,13 @@ class GithubRepoPageProcessor implements PageProcessor {
         this.strict = strict;
     }
 
+    GithubRepoPageProcessor(boolean strict, boolean backspaceHead, boolean backspaceTail) {
+        super();
+        this.strict = strict;
+        this.backspaceHead = backspaceHead;
+        this.backspaceTail = backspaceTail;
+    }
+
     @Override
     public void process(Page page) {
 
@@ -45,7 +56,9 @@ public void process(Page page) {
             emailRex = "\'(.*)@(.*)\'";
             special = true;
         }
+
         Selectable selectablem;
+
         if (operators!=null && operators.length>0){
             String atRegex = "";
             if (operators.length == 1) {
@@ -58,12 +71,31 @@ public void process(Page page) {
                     }
                 }
             }
+            if (backspaceHead) {
+                atRegex = "\\s*" + atRegex;
+            }
+
+            if (backspaceHead) {
+                atRegex = atRegex + "\\s*";
+            }
+
             selectablem = page.getHtml().replace(atRegex, "@");
         } else {
             selectablem = page.getHtml();
         }
+
+        /*全局匹配*/
+        /*
+        Selectable selectablem = page.getHtml();
+        if (operators!=null && operators.length>0) {
+            for (int i = 0; i < operators.length; i++) {
+                selectablem = selectablem.replace(operators[i], "@");
+            }
+        }
+        */
 // 包含@
 
+        //System.out.println(selectablem);
 //        String emailRex = "/[\\w!#$%&'*+/=?^_`{|}~-]+(?:\\.[\\w!#$%&'*+/=?^_`{|}~-]+)*@(?:[\\w](?:[\\w-]*[\\w])?\\.)+[\\w](?:[\\w-]*[\\w])?/
 
 //        page.putField("email", page.getHtml().regex(emailRex));

diff --git a/src/com/company/ScrawerUI.java b/src/com/company/ScrawerUI.java
@@ -1,4 +1,5 @@
-package com.company;/**
+package com.company;
+/**
  * Created by jason on 15/01/2017.
  */
 
@@ -48,7 +49,7 @@ public void start(Stage primaryStage) {
         }
         searchEmailBox.setSeperator(ats);
 
-        this.addCLickLisener(new CrawlerListener() {
+        searchEmailBox.addCLickLisener(new SearchEmailBoxListener() {
             @Override
             public boolean buttonClicked(boolean on) {
                 try {
@@ -64,9 +65,17 @@ public boolean buttonClicked(boolean on) {
                 }
                 return true;
             }
+
+            public void saveButtonPressedWithText(String string) {
+                try {
+                    FileHelper.writeToConfigFile(string);
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
         });
 
-        this.addSearchDomainCLickLisener(new SearchDomainBoxListener() {
+        searchDomainBox.addCLickLisener(new SearchDomainBoxListener() {
             @Override
             public boolean buttonClicked(boolean on) {
                 try {
@@ -162,9 +171,10 @@ private  boolean runP () throws IOException, InterruptedException {
             int endIndex = searchEmailBox.getEndIndex();
             boolean strictMode = searchEmailBox.enableStrictMode();
             String ats = searchEmailBox.getSeperator();
+            System.out.println("custom seperator:"+ats);
             //////////////////////////////////////////////////////////////////////////////////////
 
-            GithubRepoPageProcessor pageProcessor = new GithubRepoPageProcessor(false);
+            GithubRepoPageProcessor pageProcessor = new GithubRepoPageProcessor(false, searchEmailBox.getBackSpaceHead(), searchEmailBox.getBackSpaceTail());
             spider = Spider.create(pageProcessor);
 
             ArrayList urls = new ArrayList();
@@ -257,23 +267,15 @@ private void showAlertWithMessage(String message) {
 
 
 
-    public void startScrawling (boolean start) {
+    void startScrawling (boolean start) {
         if (getFunctionType()) {
             searchDomainBox.setWorking(start);
         } else  {
             searchEmailBox.setWorking(start);
         }
     }
 
-    public void addCLickLisener(CrawlerListener listener) {
-        searchEmailBox.getStartButton().setOnAction(new EventHandler<ActionEvent>() {
-            @Override
-            public void handle(ActionEvent event) {
-                boolean suc = listener.buttonClicked(!searchEmailBox.isWorking());
-                startScrawling(!searchEmailBox.isWorking());
-            }
-        });
-    }
+
 
     public void addSearchDomainCLickLisener(SearchDomainBoxListener listener) {
         searchDomainBox.getStartButton().setOnAction(new EventHandler<ActionEvent>() {

diff --git a/src/com/company/SearchDomainBox.java b/src/com/company/SearchDomainBox.java
@@ -24,6 +24,8 @@ interface SearchDomainBoxListener {
 
 public class SearchDomainBox extends VBox {
 
+    private SearchDomainBoxListener listener;
+
     public boolean isWorking() {
         return isWorking;
     }
@@ -100,4 +102,8 @@ public ArrayList getKeywords() {
         return arrayList;
     }
 
+    void addCLickLisener(SearchDomainBoxListener searchEmailBoxListener) {
+        listener = searchEmailBoxListener;
+    }
+
 }
diff --git a/src/com/company/SearchEmailBox.java b/src/com/company/SearchEmailBox.java
@@ -1,9 +1,16 @@
 package com.company;
 
+import com.sun.java.swing.action.SaveAction;
+import javafx.beans.value.ChangeListener;
+import javafx.beans.value.ObservableValue;
+import javafx.event.ActionEvent;
+import javafx.event.EventHandler;
 import javafx.geometry.Insets;
 import javafx.geometry.Pos;
 import javafx.scene.control.*;
+import javafx.scene.input.KeyEvent;
 import javafx.scene.layout.GridPane;
+import javafx.scene.layout.HBox;
 import javafx.scene.layout.VBox;
 import javafx.scene.paint.Color;
 import javafx.scene.text.Font;
@@ -13,17 +20,17 @@
 /**
  * Created by jason on 16/01/2017.
  */
-interface CrawlerListener {
+interface SearchEmailBoxListener {
 
     boolean buttonClicked(boolean on);
 
+    void saveButtonPressedWithText(String str);
+
 }
 
 public class SearchEmailBox extends VBox {
 
-    public Button getStartButton() {
-        return startButton;
-    }
+    private SearchEmailBoxListener listener;
 
     public boolean isWorking() {
         return isWorking;
@@ -44,10 +51,12 @@ public void setWorking(boolean working) {
     Spinner indexingSpinner2;
     RadioButton precending;
     ToggleGroup toggleGroup;
+    CheckBox backspaceCheckBox1;
+    CheckBox backspaceCheckBox2;
     CheckBox strictModeBox;
     CheckBox debugModeBox;
     private TextArea sepText;
-
+    Button saveButton;
     public SearchEmailBox(double padding) {
 
         super(padding);
@@ -92,8 +101,10 @@ public SearchEmailBox(double padding) {
 
         indexingSpinner1 = new Spinner(0, 99999999, 0, 1);
         indexingSpinner1.setMaxWidth(100);
+        indexingSpinner1.setEditable(true);
         indexingSpinner2 = new Spinner(0, 99999999, 0, 1);
         indexingSpinner2.setMaxWidth(100);
+        indexingSpinner2.setEditable(true);
         workingModePane.add(indexingText, 0,1);
         workingModePane.add(indexingSpinner1, 1, 1);
         workingModePane.add(indexingSpinner2, 2, 1);
@@ -111,23 +122,46 @@ public SearchEmailBox(double padding) {
 
         /**************** strict mode ********************/
 
-        strictModeBox = new CheckBox("开启严肃模式");
+        strictModeBox = new CheckBox("严肃模式");
         strictModeBox.setSelected(true);
         workingModePane.add(strictModeBox,0, 3);
 
         this.getChildren().add(workingModePane);
 
-
         /**************** revise code ********************/
 
         Text revise = new Text("添加@替换,逗号分隔");
         this.getChildren().add(revise);
 
+        GridPane reviseCodePane = new GridPane();
+        reviseCodePane.setHgap(30);
+        backspaceCheckBox1 = new CheckBox("@前匹配空格或回车");
+        backspaceCheckBox1.setSelected(false);
+        reviseCodePane.add(backspaceCheckBox1, 0, 0); //\s*
+
+        backspaceCheckBox2 = new CheckBox("@后匹配空格或回车");
+        backspaceCheckBox2.setSelected(false);
+        reviseCodePane.add(backspaceCheckBox2, 1, 0); //\s*
+
+        this.getChildren().add(reviseCodePane);
+
+
         sepText = new TextArea();
         sepText.setWrapText(true);
-        sepText.setPrefSize(300, 50);
         sepText.setMaxSize(Double.MAX_VALUE, Double.MAX_VALUE);
-        this.getChildren().add(sepText);
+        sepText.textProperty().addListener(new ChangeListener<String>() {
+            @Override
+            public void changed(final ObservableValue<? extends String> observable, final String oldValue, final String newValue) {
+                saveButton.setDisable(false);
+            }
+        });
+
+        saveButton = new Button("save");
+        saveButton.setMinSize(60, 60);
+        saveButton.setDisable(true);
+        HBox hbox = new HBox(10,sepText, saveButton);
+        hbox.setPrefSize(300, 60);
+        this.getChildren().add(hbox);
 
         debugModeBox = new CheckBox("开启debug模式");
         debugModeBox.setSelected(false);
@@ -140,6 +174,28 @@ public SearchEmailBox(double padding) {
         startButton.setMaxWidth(Double.MAX_VALUE);
         this.getChildren().add(startButton);
 
+        startButton.setOnAction(new EventHandler<ActionEvent>() {
+            @Override
+            public void handle(ActionEvent event) {
+                boolean suc = listener.buttonClicked(!isWorking);
+                setWorking(suc);
+            }
+        });
+
+        saveButton.setOnAction(new EventHandler<ActionEvent>() {
+            @Override
+            public void handle(ActionEvent event) {
+                listener.saveButtonPressedWithText(sepText.getText());
+                saveButton.setDisable(true);
+            }
+        });
+
+
+
+    }
+
+    void addCLickLisener(SearchEmailBoxListener searchEmailBoxListener) {
+        listener = searchEmailBoxListener;
     }
 
 
@@ -175,6 +231,14 @@ public String getSeperator() {
         return sepText.getText();
     }
 
+    public boolean getBackSpaceHead() {
+        return backspaceCheckBox1.isSelected();
+    }
+
+    public boolean getBackSpaceTail() {
+        return backspaceCheckBox2.isSelected();
+    }
+
     public boolean isEnableDebug() {
         return debugModeBox.isSelected();
     }