Modernize

.github/workflows/python-arabic.yml

@@ -1,9 +1,24 @@
-name: python
+name: Python Arabic CI
 
 on:
   push:
     branches: [ main ]
+    paths:
+      - 'python/**'
+      - '.github/workflows/python-arabic.yml'
   pull_request:
+    paths:
+      - 'python/**'
+      - '.github/workflows/python-arabic.yml'
+
+# Cancel in-progress runs for the same workflow and branch
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  security-events: write
 
 jobs:
   lint:
@@ -17,74 +32,132 @@ jobs:
       - run: ruff check python/
       - run: ruff format --check python/
 
+  dependency-review:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+
+      - name: Dependency Review
+        uses: actions/dependency-review-action@v3
+
   infer:
     runs-on: ubuntu-latest
+    # cold pip cache: the eager torch wheel is ~5GB
+    timeout-minutes: 20
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9"]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
-      - uses: actions/checkout@v7
-      - uses: actions/setup-python@v7
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: pip
-          cache-dependency-path: python/arabic/requirements.txt
-
-      - name: Install requirements
-        working-directory: ./python/arabic
-        run: |
-          pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
-
-      - name: Download PyTorch model
-        working-directory: ./python/arabic
-        run: |
-          curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt \
-            -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt
-
-      - name: Run diacriticization
-        working-directory: ./python/arabic
-        run: |
-          python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر'
+    - name: Checkout code
+      uses: actions/checkout@v7
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v7
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          python/arabic/requirements.txt
+          python/arabic/setup.py
+
+    - name: Install requirements
+      working-directory: ./python/arabic
+      run: |
+        python -m pip install --upgrade pip
+        pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
+
+    - name: Create model directory
+      working-directory: ./python/arabic
+      run: |
+        mkdir -p log_dir/CA_MSA.base.cbhg/models/
+
+    - name: Download PyTorch model
+      working-directory: ./python/arabic
+      run: |
+        curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt \
+          -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt
+
+    - name: Run diacriticization
+      working-directory: ./python/arabic
+      run: |
+        python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر'
 
   train:
     runs-on: ubuntu-latest
+    # the WIP 51-step CPU train needs ~35 min after dataset download
+    timeout-minutes: 60
     env:
       # tensorboard's generated descriptors predate protobuf 4; the pure-python
       # implementation tolerates them without pinning protobuf
       PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9"]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
-      - uses: actions/checkout@v7
-      - uses: actions/setup-python@v7
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: pip
-          cache-dependency-path: python/arabic/requirements.txt
-
-      - name: Install requirements
-        working-directory: ./python/arabic
-        run: |
-          pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
-
-      - name: Prepare dataset
-        working-directory: ./python/arabic
-        run: |
-          mkdir -p data/CA_MSA
-          touch data/CA_MSA/{eval,train,test}.csv
-          cd data
-          curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip
-          unzip tashkeela.zip
-          for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done
-          for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done
-          for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done
-
-      - name: Try training (WIP)
-        working-directory: ./python/arabic
-        run: |
-          python train.py --model "cbhg" --config config/test_cbhg.yml
+    - name: Checkout code
+      uses: actions/checkout@v7
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v7
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          python/arabic/requirements.txt
+          python/arabic/setup.py
+
+    - name: Install requirements
+      working-directory: ./python/arabic
+      run: |
+        python -m pip install --upgrade pip
+        pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
+
+    - name: Prepare dataset
+      working-directory: ./python/arabic
+      run: |
+        mkdir -p data/CA_MSA
+        touch data/CA_MSA/{eval,train,test}.csv
+        cd data
+        curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip
+        unzip tashkeela.zip
+        for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done
+        for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done
+        for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done
+
+    - name: Try training (WIP)
+      working-directory: ./python/arabic
+      run: |
+        python train.py --model "cbhg" --config config/test_cbhg.yml
+
+  security-scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v7
+
+    - name: Set up Python
+      uses: actions/setup-python@v7
+      with:
+        python-version: '3.11'
+
+    - name: Install safety
+      run: |
+        python -m pip install --upgrade pip
+        pip install safety
+
+    - name: Check for vulnerabilities in Arabic requirements
+      working-directory: ./python/arabic
+      run: |
+        safety check -r requirements.txt --full-report || true
+
+    - name: Check for vulnerabilities in Hebrew requirements
+      working-directory: ./python/hebrew
+      run: |
+        safety check -r requirements.txt --full-report || true

.github/workflows/python-hebrew.yml

@@ -0,0 +1,110 @@
+name: Python Hebrew CI
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'python/hebrew/**'
+      - '.github/workflows/python-hebrew.yml'
+  pull_request:
+    paths:
+      - 'python/hebrew/**'
+      - '.github/workflows/python-hebrew.yml'
+
+# Cancel in-progress runs for the same workflow and branch
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+
+      - name: Dependency Review
+        uses: actions/dependency-review-action@v3
+
+  infer:
+    runs-on: ubuntu-latest
+    # cold pip cache: the eager torch wheel is ~5GB
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v7
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v7
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          python/hebrew/requirements.txt
+          python/hebrew/setup.py
+
+    - name: Install requirements
+      working-directory: ./python/hebrew
+      run: |
+        python -m pip install --upgrade pip
+        pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
+
+    - name: Create model directory
+      working-directory: ./python/hebrew
+      run: |
+        mkdir -p log_dir/base.cbhg/
+
+    - name: Run diacriticization
+      working-directory: ./python/hebrew
+      run: |
+        python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'שלום'
+
+  train:
+    runs-on: ubuntu-latest
+    # the WIP 51-step CPU train needs ~36 min after install
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v7
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v7
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          python/hebrew/requirements.txt
+          python/hebrew/setup.py
+
+    - name: Install requirements
+      working-directory: ./python/hebrew
+      run: |
+        python -m pip install --upgrade pip
+        pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
+
+    - name: Prepare test data
+      working-directory: ./python/hebrew
+      run: |
+        mkdir -p data/train data/eval
+        printf "שלום עולם\nשלום עולם\nשלום עולם\nשלום עולם\nשלום עולם\n" > data/train/test.txt
+        printf "שלום עולם\nשלום עולם\n" > data/eval/test.txt
+
+    - name: Try training (WIP)
+      working-directory: ./python/hebrew
+      run: |
+        python train.py --model "cbhg" --config config/test_cbhg.yml