rababa/.github/workflows/python-arabic.yml at main · interscript/rababa · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
name: Python Arabic CI

on:
  push:
    branches: [ main ]
    paths:
      - 'python/**'
      - '.github/workflows/python-arabic.yml'
  pull_request:
    paths:
      - 'python/**'
      - '.github/workflows/python-arabic.yml'

# Cancel in-progress runs for the same workflow and branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  security-events: write

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.11"
      - run: pip install ruff
      - run: ruff check python/
      - run: ruff format --check python/

  dependency-review:
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Dependency Review
        uses: actions/dependency-review-action@v3

  infer:
    runs-on: ubuntu-latest
    # cold pip cache: the eager torch wheel is ~5GB
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.9', '3.10', '3.11', '3.12']

    steps:
    - name: Checkout code
      uses: actions/checkout@v7

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v7
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
        cache-dependency-path: |
          python/arabic/requirements.txt
          python/arabic/setup.py

    - name: Install requirements
      working-directory: ./python/arabic
      run: |
        python -m pip install --upgrade pip
        pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .

    - name: Create model directory
      working-directory: ./python/arabic
      run: |
        mkdir -p log_dir/CA_MSA.base.cbhg/models/

    - name: Download PyTorch model
      working-directory: ./python/arabic
      run: |
        curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt \
          -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt

    - name: Run diacriticization
      working-directory: ./python/arabic
      run: |
        python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر'

  train:
    runs-on: ubuntu-latest
    # the WIP 51-step CPU train needs ~35 min after dataset download
    timeout-minutes: 60
    env:
      # tensorboard's generated descriptors predate protobuf 4; the pure-python
      # implementation tolerates them without pinning protobuf
      PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.9', '3.10', '3.11', '3.12']

    steps:
    - name: Checkout code
      uses: actions/checkout@v7

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v7
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
        cache-dependency-path: |
          python/arabic/requirements.txt
          python/arabic/setup.py

    - name: Install requirements
      working-directory: ./python/arabic
      run: |
        python -m pip install --upgrade pip
        pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .

    - name: Prepare dataset
      working-directory: ./python/arabic
      run: |
        mkdir -p data/CA_MSA
        touch data/CA_MSA/{eval,train,test}.csv
        cd data
        curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip
        unzip tashkeela.zip
        for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done
        for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done
        for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done

    - name: Try training (WIP)
      working-directory: ./python/arabic
      run: |
        python train.py --model "cbhg" --config config/test_cbhg.yml

  security-scan:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
    - name: Checkout code
      uses: actions/checkout@v7

    - name: Set up Python
      uses: actions/setup-python@v7
      with:
        python-version: '3.11'

    - name: Install safety
      run: |
        python -m pip install --upgrade pip
        pip install safety

    - name: Check for vulnerabilities in Arabic requirements
      working-directory: ./python/arabic
      run: |
        safety check -r requirements.txt --full-report || true

    - name: Check for vulnerabilities in Hebrew requirements
      working-directory: ./python/hebrew
      run: |
        safety check -r requirements.txt --full-report || true