Skip to content

Commit

Permalink
2
Browse files Browse the repository at this point in the history
  • Loading branch information
Cryolitia committed Dec 28, 2024
1 parent 9840afd commit 23d8e0e
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 59 deletions.
1 change: 0 additions & 1 deletion .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
FROM konghayao123/best-cn-dev-container:1.1.0
RUN apt install -y llvm clang pkg-config libssl-dev
RUN apt install -y protobuf-compiler && protoc --version
RUN apt install -y opencc && opencc --version
RUN bash /init/rust.sh
ENV RUSTUP_DIST_SERVER="https://rsproxy.cn"
ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup"
Expand Down
11 changes: 3 additions & 8 deletions .github/workflows/cross-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,17 +85,12 @@ jobs:
scoop install main/mingw
scoop install main/nodejs
pip install opencc --target $HOME
echo "OPENCC_BIN=$HOME/opencc/clib/bin/opencc" | Out-File -FilePath $env:GITHUB_ENV -Append
echo "OPENCC_DATA_PATH=$HOME/opencc/clib/share/opencc/" | Out-File -FilePath $env:GITHUB_ENV -Append
$env:Path += ";$HOME/opencc/clib/bin"
opencc --version
${{ matrix.platform.setup }}
- name: Setup Linux Environment
if: ${{ runner.os == 'Linux' }}
run: |
sudo apt update
sudo apt install -y nodejs llvm clang pkg-config libssl-dev opencc
sudo apt install -y nodejs llvm clang pkg-config libssl-dev
${{ matrix.platform.setup }}
- name: Set up Homebrew
id: set-up-homebrew
Expand All @@ -105,7 +100,7 @@ jobs:
- name: Setup MacOS Environment
if: ${{ runner.os == 'macOS' }}
run: |
brew install llvm protobuf opencc automake libtool
brew install llvm protobuf automake libtool
${{ matrix.platform.setup }}
- name: Install Protoc
Expand Down Expand Up @@ -175,7 +170,7 @@ jobs:
run: |
sudo apt update
sudo apt upgrade
sudo apt install -y llvm clang pkg-config libssl-dev protobuf-compiler opencc
sudo apt install -y llvm clang pkg-config libssl-dev protobuf-compiler
- name: Setup WASI Environment
working-directory: .
Expand Down
120 changes: 120 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 0 additions & 7 deletions Developer.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,6 @@ apt update -y
apt install -y protobuf-compiler && protoc --version
```

## 安装 opencc 命令

```sh
apt update -y
apt install -y opencc && opencc --version
```

## 安装 Wasm-sdk

安装并自动配置,执行文件在 .devcontainer/wasi-install.sh,需要在容器中执行
Expand Down
4 changes: 4 additions & 0 deletions crates/lang_unicodes/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,7 @@ lazy_static = "1.5.0"

[lib]
path = "src/lib.rs"

[build-dependencies]
lazy_static = "1.5.0"
opencc-rs = "0.4.7"
54 changes: 11 additions & 43 deletions crates/lang_unicodes/build.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use lazy_static::lazy_static;
use opencc_rs::{Config, OpenCC};
use std::collections::HashSet;
use std::fs::{create_dir, exists, read_to_string};
use std::io::Write;
use std::process::{Command, Stdio};
use std::{env, fs};

const CN_SYMBOL: &str = "⸺、。〈〉《》「」『』【】〔〕〖〗︐︑︒︓︔︕︖︐︑︒︓︔︕︖︗︘︙︰︱︳︴︵︶︷︸︹︺︻︼︽︾︿﹀﹁﹂﹃﹄";
Expand All @@ -15,28 +15,12 @@ fn encode_utf16(s: &char) -> u16 {
*s.encode_utf16(&mut buf).iter().next().unwrap()
}

fn opencc_convert(
s: String,
opencc_bin: &str,
opencc_data_path: &Option<String>,
) -> String {
let mut binding = Command::new(opencc_bin);
let mut command = binding
.arg("-c")
.arg("s2t.json")
.stdin(Stdio::piped())
.stdout(Stdio::piped());
if let Some(data_path) = opencc_data_path {
command = command.arg("--path").arg(data_path);
}
let mut child = command.spawn().unwrap();

let mut stdin = child.stdin.take().unwrap();
stdin.write_all(s.as_bytes()).unwrap();
drop(stdin);
lazy_static! {
static ref OPENCC: OpenCC = OpenCC::new([Config::S2T]).unwrap();
}

let output = child.wait_with_output().unwrap();
String::from_utf8_lossy(&output.stdout).to_string()
fn opencc_convert(s: String) -> String {
OPENCC.convert(s).unwrap()
}

/*
Expand Down Expand Up @@ -75,17 +59,12 @@ fn opencc_convert(
*/

fn process_chinese_chars(opencc_bin: &str, opencc_data: &Option<String>) {
fn process_chinese_chars() {
let sc: Vec<char> =
CN_SYMBOL.chars().chain(HAN_ZI_PIN_LV.chars()).clone().collect();
let tc: Vec<u16> = sc
.iter()
.map(|i| {
opencc_convert(i.to_string(), opencc_bin, opencc_data)
.chars()
.next()
.unwrap()
})
.map(|i| opencc_convert(i.to_string()).chars().next().unwrap())
.map(|i| encode_utf16(&i))
.collect();
let sc: Vec<u16> = sc.iter().map(encode_utf16).collect();
Expand Down Expand Up @@ -156,6 +135,7 @@ fn main() {
println!("cargo::rerun-if-changed={}", CN_CHAR_RANK_FILE);
println!("cargo::rerun-if-changed={}", HANGUL_SYL_FILE);
println!("cargo::rerun-if-env-changed=CARGO_WITH_NO_EXTRA");

if !exists("./data").unwrap() {
create_dir("./data").unwrap();
}
Expand All @@ -171,18 +151,6 @@ fn main() {
}
}

let opencc_bin: String =
env::var("OPENCC_BIN").unwrap_or("opencc".to_string());
let opencc_data: Option<String> = env::var("OPENCC_DATA_PATH").ok();
// 检查 opencc 是否存在
let status = Command::new(&opencc_bin).arg("--version").status();
if status.is_err() {
panic!(
"opencc: {} is not installed or not in PATH \n {:?}",
opencc_bin, status
);
}

process_chinese_chars(&opencc_bin, &opencc_data);
process_chinese_chars();
process_korean_syllables();
}

0 comments on commit 23d8e0e

Please sign in to comment.