From 3deee635a33c2c7fc855844af2168238f5050804 Mon Sep 17 00:00:00 2001 From: sahithya Date: Thu, 23 Dec 2021 00:40:45 -0800 Subject: [PATCH 01/11] new website v1 --- docker-compose.yml | 41 ++++++++++++++++++++++++++--------------- readme.md | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3dacf2d..6d1fc3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,13 +15,31 @@ services: # MySQL dbs mysql: image: mysql:5.7.25 + container_name: mysql_test + platform: linux/x86_64 + ports: + # : < MySQL Port running inside container> + - '3307:3306' command: mysqld --default-authentication-plugin=mysql_native_password build: ./mysql environment: - MYSQL_ROOT_PASSWORD: "" + MYSQL_ROOT_PASSWORD: "1993sahi11" MYSQL_DATABASE: openml - + + + website_new: + build: openml.org + ports: + - "5000:5000" + links: + - mysql + + depends_on: + - mysql + - elasticsearch + + # Elastic search indexes elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:6.8.2 @@ -31,23 +49,22 @@ services: - "9200:9200" - "9300:9300" - # Php frontend +# Php frontend website: build: ./website ports: - 80:80 - volumes: + volumes: - ./config:/openmlconfig - ./OpenML:/var/www/html depends_on: - mysql - elasticsearch - - gollum - + # MySQL admin UI (tool) phpmyadmin: image: phpmyadmin/phpmyadmin:latest - volumes: + volumes: - ./config:/openmlconfig ports: - 8080:80 @@ -59,15 +76,9 @@ services: # OpenML Java background workers java: image: java - volumes: + volumes: - ./config:/openmlconfig - ./OpenML:/var/www/html - + # runs openml process_dataset java background task each 10 secs command: /bin/bash -c 'while sleep 10;do api_key=`cat /openmlconfig/api_key.txt`; echo "API key:$$api_key"; java -Xmx4G -jar /var/www/html/openml_OS/third_party/OpenML/Java/evaluate.jar -config "server = http://website/;api_key = $$api_key" -f process_dataset;done' - - # OpenML Wiki, TODO: not working , CORS/cross domain issue - gollum: - build: ./gollum - ports: - - 9080:80 diff --git a/readme.md b/readme.md index 713176f..13fda0f 100644 --- a/readme.md +++ b/readme.md @@ -33,6 +33,21 @@ git clone -b docker_changes https://github.com/openml/OpenML.git ``` ![](images/2018-04-07-00-57-29.png) +#### New website configuration (Optional) +Skip this step unless you want to use the new website. + +- Clone the new website from here, as it has some code changes +``` +git clone -b docker https://github.com/PortML/openml.org.git +cd openml.org + +``` +- Edit DATABASE_URI field in *.flaskenv* to add in the mysql password in place of PASSWORD- + (use the same password as the mysql password in docker-compose.yml) +- Continue with remaining steps and view Step 7 for testing +new website changes + + ### Step 2: Configure docker and OpenML @@ -75,6 +90,9 @@ Disable email activation in *OpenML\openml_OS\ion_auth.php* ![](images/2018-04-07-01-07-21.png) + + + ### Step 3: Starting docker-compose On the openml-docker-dev root folder, where *docker-compose.yml* is located run: @@ -145,3 +163,10 @@ We have 1 sample dataset ![](images/upload2.PNG) ### Note: Files in OpenML cloned repo are mounted inside the website container, any change will reflect immediately on the site + +### Step 7 New website checks (Optional) +- Check the new website running at 127.0.0.1/5000. It should look similar to new.openml.org +- Sign up as a new user in the new website. (Note that you cannot use the admin account from the old website to login here) +- Sign in with your email and password +- You should be able to see your profile +- Dataset upload is not working yet (some connection issue in docker compose - in progress) \ No newline at end of file From faced5d0082dbbac60f3ed934b089e0ab27f3347 Mon Sep 17 00:00:00 2001 From: Sahithya Ravi <44670788+sahithyaravi1493@users.noreply.github.com> Date: Thu, 23 Dec 2021 00:49:12 -0800 Subject: [PATCH 02/11] Update readme.md --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 13fda0f..9f339fc 100644 --- a/readme.md +++ b/readme.md @@ -36,7 +36,7 @@ git clone -b docker_changes https://github.com/openml/OpenML.git #### New website configuration (Optional) Skip this step unless you want to use the new website. -- Clone the new website from here, as it has some code changes +- Clone the new website from here (also inside openml-docker-dev), as it has some code changes ``` git clone -b docker https://github.com/PortML/openml.org.git cd openml.org @@ -169,4 +169,4 @@ We have 1 sample dataset - Sign up as a new user in the new website. (Note that you cannot use the admin account from the old website to login here) - Sign in with your email and password - You should be able to see your profile -- Dataset upload is not working yet (some connection issue in docker compose - in progress) \ No newline at end of file +- Dataset upload is not working yet (some connection issue in docker compose - in progress) From 4ba87b46dbdfb5bba363835111e4ce0e57837c54 Mon Sep 17 00:00:00 2001 From: Sahithya Ravi <44670788+sahithyaravi1493@users.noreply.github.com> Date: Thu, 23 Dec 2021 00:49:30 -0800 Subject: [PATCH 03/11] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6d1fc3b..7330ece 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ services: command: mysqld --default-authentication-plugin=mysql_native_password build: ./mysql environment: - MYSQL_ROOT_PASSWORD: "1993sahi11" + MYSQL_ROOT_PASSWORD: "" MYSQL_DATABASE: openml From 69bd094980529ef0bc105bf1814af1f09ed28d1e Mon Sep 17 00:00:00 2001 From: sahithya Date: Wed, 19 Jan 2022 08:04:01 -0800 Subject: [PATCH 04/11] read me update --- readme.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 9f339fc..c3f0663 100644 --- a/readme.md +++ b/readme.md @@ -40,15 +40,31 @@ Skip this step unless you want to use the new website. ``` git clone -b docker https://github.com/PortML/openml.org.git cd openml.org +cp server/src/client/app/TEMPLATE.env .env ``` - Edit DATABASE_URI field in *.flaskenv* to add in the mysql password in place of PASSWORD- (use the same password as the mysql password in docker-compose.yml) -- Continue with remaining steps and view Step 7 for testing -new website changes - +- Note on DATABASE-URI: It seems like mysql-test works for MAC OS and localhost:3306 for windows. +- Check openml.org/server/src/client/app/.env if the react url is correct +- Please make sure you rebuild the openml.org docker image if you make any changes to these configuration files using: +- This will make sure the react image is re-built: +``` +cd openml.org +docker build -t openml-docker -f Dockerfile . +``` +- Continue with remaining steps and view Step 7 for testing new website changes +- Switch back to root folder +``` +cd .. +``` +If the new website rebuild doesn't work, try clearing all caches with: +``` +docker system prune -a +``` +and then, pull elastic search before doing docker-compose up. ### Step 2: Configure docker and OpenML Edit *docker-compose.yml* mainly define a secure **mysql password**: From 50a5220fcd775c1f268e478edfe695f70eeca7a8 Mon Sep 17 00:00:00 2001 From: MatsKolsteeg Date: Wed, 2 Mar 2022 09:08:00 +0100 Subject: [PATCH 05/11] Removed quotation marks in http.cors.allow-headers: which are preventing the correct CORS policy to be set --- elasticsearch/elasticsearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/elasticsearch.yml b/elasticsearch/elasticsearch.yml index 7b8be73..78c1053 100644 --- a/elasticsearch/elasticsearch.yml +++ b/elasticsearch/elasticsearch.yml @@ -8,4 +8,4 @@ http.host: 0.0.0.0 http.cors.enabled: true http.cors.allow-origin: "*" http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE -http.cors.allow-headers: "X-Requested-With,X-Auth-Token,Content-Type, Content-Length, Authorization" \ No newline at end of file +http.cors.allow-headers: X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization From 08ba2f825f57320f603bcddceb41e491ef2ffd4e Mon Sep 17 00:00:00 2001 From: Joost van der Gaag Date: Fri, 11 Mar 2022 10:18:30 +0100 Subject: [PATCH 06/11] Updated ReadMe and elasticSearch Dockerfile (added specific version) --- elasticsearch/Dockerfile | 2 +- readme.md | 70 ++++++++++++++++++++++++++-------------- 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 9169c69..c30792e 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -1,4 +1,4 @@ -FROM elasticsearch +FROM elasticsearch:6.8.2 LABEL maintainer="Rui Quintino,https://github.com/rquintino" COPY ./elasticsearch.yml /usr/share/elasticsearch/config/ \ No newline at end of file diff --git a/readme.md b/readme.md index c3f0663..f1c4467 100644 --- a/readme.md +++ b/readme.md @@ -43,28 +43,7 @@ cd openml.org cp server/src/client/app/TEMPLATE.env .env ``` -- Edit DATABASE_URI field in *.flaskenv* to add in the mysql password in place of PASSWORD- - (use the same password as the mysql password in docker-compose.yml) -- Note on DATABASE-URI: It seems like mysql-test works for MAC OS and localhost:3306 for windows. -- Check openml.org/server/src/client/app/.env if the react url is correct -- Please make sure you rebuild the openml.org docker image if you make any changes to these configuration files using: -- This will make sure the react image is re-built: -``` -cd openml.org -docker build -t openml-docker -f Dockerfile . -``` -- Continue with remaining steps and view Step 7 for testing new website changes -- Switch back to root folder -``` -cd .. - -``` -If the new website rebuild doesn't work, try clearing all caches with: -``` -docker system prune -a -``` -and then, pull elastic search before doing docker-compose up. ### Step 2: Configure docker and OpenML Edit *docker-compose.yml* mainly define a secure **mysql password**: @@ -106,7 +85,47 @@ Disable email activation in *OpenML\openml_OS\ion_auth.php* ![](images/2018-04-07-01-07-21.png) +#### New website configuration +- Edit DATABASE_URI field in *.flaskenv* to add in the mysql password in place of PASSWORD- + (use the same password as the mysql password in docker-compose.yml) +- Note on DATABASE_URI: hostname should be 'mysql_test', the container name of database: + ``DATABASE_URI=mysql+pymysql://[username]:[password]@mysql_test:3306/openml`` +- Check openml.org/server/src/client/app/.env if the react url is correct +- In order to enable the python debug prints in docker add the following lines of code to the main 'docker-compose.yml' file inside the 'website_new' service +```` +environment: + - PYTHONUNBUFFERED=1 +```` +- Please make sure you rebuild the openml.org docker image if you make any changes to these configuration files using: + This will make sure the react image is re-built: +``` +cd openml.org +docker build -t openml-docker -f Dockerfile . +``` +- [Optionally] you can use a hot-reload configuration for the new website with some constrains, see the section below how to set this up + +- Continue with remaining steps and view Step 7 for testing new website changes +- Switch back to root folder + + +``` +cd .. + +``` +If the new website rebuild doesn't work, try clearing all caches with: +``` +docker system prune -a +``` +and then, pull elastic search before doing docker-compose up. + +### [Optional hot-reload new website] +Using a hot-reload in docker requires you to set the volume of the source code to your local folder of the new website. Flask runs in development mode and will see changes you make without requireing you to rebuild the image. The only downside is that you are unable to reach the new front-end (React code) via the docker URL. You can seperatly run a node development server for the front-end to also enable hot-reload for the React front-end. +Add the following lines of code to the main 'docker-compose.yml' file inside the 'website_new' service to enable this hot-reload function: +```` +volumes: + - ./openml.org:/app +```` ### Step 3: Starting docker-compose @@ -139,7 +158,7 @@ after start wait a few seconds for services to be ready, ex: MySQL ready for con Execute in a new window/shell: ``` -docker exec -it openmldockerdev_website_1 php index.php cron init_local_env +docker exec -it openml-docker-dev_website_1 php index.php cron init_local_env ``` (take note the printed admin username and password, and wait to finish, can take 1-2mins) @@ -152,12 +171,12 @@ Change data folder owner to www-data apache user in container, allow for logs/up Execute in a new window/shell: ``` -docker exec -it openmldockerdev_website_1 chown -R www-data:www-data /var/www/html/data +docker exec -it openml-docker-dev_website_1 chown -R www-data:www-data /var/www/html/data ``` -### Step 6: Final tests +### Step 6: Final tests (Old website) Login on http://localhost with admin and saved password @@ -185,4 +204,5 @@ We have 1 sample dataset - Sign up as a new user in the new website. (Note that you cannot use the admin account from the old website to login here) - Sign in with your email and password - You should be able to see your profile -- Dataset upload is not working yet (some connection issue in docker compose - in progress) +- By default the user created above is not an admin. This is required if you want to use the dataset upload. This can be done by loggin into MyPHPAdmin and changing the 'users_groups' row of this user. Set the 'group_id' to 1 (admin group) and save. +- Check Dataset upload (required to fill in all fields) From 29c6f8b6c3f0f8e0100313178fb8ac25d9470341 Mon Sep 17 00:00:00 2001 From: Mats Kolsteeg <94629828+MatsPortML@users.noreply.github.com> Date: Fri, 11 Mar 2022 11:08:38 +0100 Subject: [PATCH 07/11] Update readme.md --- readme.md | 1 - 1 file changed, 1 deletion(-) diff --git a/readme.md b/readme.md index f1c4467..29bad69 100644 --- a/readme.md +++ b/readme.md @@ -6,7 +6,6 @@ # Issues/limitations - current only works on port 80 - wiki powered by gollum is not working (in the new OpenML frontend this will no longer be used). -- **Only the local admin account works for now.** There is an issue with login and it will not be possible to create and login with other accounts (Work in progress) # Requirements From 9278b48e4a8c7628ce34290c364525d011b68783 Mon Sep 17 00:00:00 2001 From: prabhant Date: Fri, 18 Mar 2022 16:21:07 +0100 Subject: [PATCH 08/11] typo fixes --- readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 29bad69..95257ea 100644 --- a/readme.md +++ b/readme.md @@ -80,7 +80,7 @@ Configure elastic search. ![](images/2018-04-07-01-04-02.png) -Disable email activation in *OpenML\openml_OS\ion_auth.php* +Disable email activation in *OpenML/openml_OS/config/ion_auth.php* ![](images/2018-04-07-01-07-21.png) @@ -157,7 +157,7 @@ after start wait a few seconds for services to be ready, ex: MySQL ready for con Execute in a new window/shell: ``` -docker exec -it openml-docker-dev_website_1 php index.php cron init_local_env +docker exec -it openml-docker-dev-website-1 php index.php cron init_local_env ``` (take note the printed admin username and password, and wait to finish, can take 1-2mins) @@ -170,7 +170,7 @@ Change data folder owner to www-data apache user in container, allow for logs/up Execute in a new window/shell: ``` -docker exec -it openml-docker-dev_website_1 chown -R www-data:www-data /var/www/html/data +docker exec -it openml-docker-dev-website-1 chown -R www-data:www-data /var/www/html/data ``` From cc116e2c343c392ec5ce5cc93c106df87ff8cbe4 Mon Sep 17 00:00:00 2001 From: Mats Kolsteeg <94629828+MatsPortML@users.noreply.github.com> Date: Mon, 21 Mar 2022 17:27:13 +0100 Subject: [PATCH 09/11] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 95257ea..894f773 100644 --- a/readme.md +++ b/readme.md @@ -96,7 +96,7 @@ environment: - PYTHONUNBUFFERED=1 ```` - Please make sure you rebuild the openml.org docker image if you make any changes to these configuration files using: - This will make sure the react image is re-built: + This will make sure the react image is re-built (Don't do this before the first docker-compose up): ``` cd openml.org docker build -t openml-docker -f Dockerfile . From ccd0161b908c9b48f0497f4468f7ca8ad1c80ca3 Mon Sep 17 00:00:00 2001 From: MatsKolsteeg Date: Wed, 1 Jun 2022 17:34:55 +0200 Subject: [PATCH 10/11] Moved command to set default-authentication-plugin = mysql_native_password from docker-compose.yml --- docker-compose.yml | 1 - mysql/Dockerfile | 4 +++- mysql/custom-mysql.cnf | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 mysql/custom-mysql.cnf diff --git a/docker-compose.yml b/docker-compose.yml index 7330ece..84089e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,6 @@ services: ports: # : < MySQL Port running inside container> - '3307:3306' - command: mysqld --default-authentication-plugin=mysql_native_password build: ./mysql environment: MYSQL_ROOT_PASSWORD: "" diff --git a/mysql/Dockerfile b/mysql/Dockerfile index 7a7a4f3..cbe6fe4 100644 --- a/mysql/Dockerfile +++ b/mysql/Dockerfile @@ -1,7 +1,9 @@ -FROM mysql +FROM mysql:5.7.25 LABEL maintainer="Rui Quintino,https://github.com/rquintino" # *.sql files in /docker-entrypoint-initdb.d are executed at mysql instance start, by alphabetical order ADD ./openml_expdb.sql /docker-entrypoint-initdb.d ADD ./openml.sql /docker-entrypoint-initdb.d +COPY ./custom-mysql.cnf /etc/mysql/conf.d/ + diff --git a/mysql/custom-mysql.cnf b/mysql/custom-mysql.cnf new file mode 100644 index 0000000..dd829be --- /dev/null +++ b/mysql/custom-mysql.cnf @@ -0,0 +1,2 @@ +[mysqld] +default-authentication-plugin = mysql_native_password \ No newline at end of file From a24fba6a55260382085ab089feff1795cac28626 Mon Sep 17 00:00:00 2001 From: MatsKolsteeg Date: Wed, 1 Jun 2022 17:37:52 +0200 Subject: [PATCH 11/11] Added environment variables for website to docker-compose.yml --- docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 84089e5..b39ef5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,6 +59,12 @@ services: depends_on: - mysql - elasticsearch + environment: + DB_HOST_OPENML: "mysql" + DB_HOST_EXPDB: "mysql" + ES_URL: 'localhost:9200' + ES_PUBLIC_URL: 'elasticsearch:9200' + BASE_URL: 'http://localhost/' # MySQL admin UI (tool) phpmyadmin: