From 4e33f9626f44316113ab07a226900a3484ada8d2 Mon Sep 17 00:00:00 2001 From: David Corne Date: Mon, 6 Jul 2020 17:06:02 +0100 Subject: [PATCH 01/12] Add a dockerfile --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..161a58e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:trusty +RUN sudo apt-get -y update +RUN sudo apt-get -y upgrade +RUN sudo apt-get install -y sqlite3 libsqlite3-dev python3 python3-pip +RUN pip3 install 'pyyaml>5.1' From 5bce34d195f7b3fe1b4f6a0a6fd3967481e2659b Mon Sep 17 00:00:00 2001 From: David Corne Date: Mon, 6 Jul 2020 17:06:42 +0100 Subject: [PATCH 02/12] On Ubuntu paths don't work in os.chdir, use strings --- bin/gendb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/gendb.py b/bin/gendb.py index 24ad979..ae8428c 100644 --- a/bin/gendb.py +++ b/bin/gendb.py @@ -11,8 +11,8 @@ import pathlib DBFILE = 'tmp.db' -DBOUTPUT_PATH = pathlib.Path().absolute().parent -DATA_PATH = pathlib.Path().absolute().parent / 'data' +DBOUTPUT_PATH = str(pathlib.Path().absolute().parent) +DATA_PATH = str(pathlib.Path().absolute().parent / 'data') def main(): From 7d4f8ad7a6d05f8627f21f473c774e4397200e2f Mon Sep 17 00:00:00 2001 From: David Corne Date: Tue, 7 Jul 2020 13:27:07 +0100 Subject: [PATCH 03/12] Try to read the yaml as binary --- bin/gendb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/gendb.py b/bin/gendb.py index ae8428c..1587498 100644 --- a/bin/gendb.py +++ b/bin/gendb.py @@ -49,7 +49,7 @@ def main(): print("{}".format(e)) # Load in the yaml data - with open('basics.yaml') as yl: + with open('basics.yaml', 'rb') as yl: data = yaml.full_load(yl) # call the functions to input to SQL do_abilityscore(data['abilityscore'], conn) From 131f32ee2a2707a2b4e30b3265487cf6fb692cd8 Mon Sep 17 00:00:00 2001 From: David Corne Date: Tue, 7 Jul 2020 13:30:14 +0100 Subject: [PATCH 04/12] Run gendb.py in the Dockerfile --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 161a58e..d4b8330 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,3 +3,9 @@ RUN sudo apt-get -y update RUN sudo apt-get -y upgrade RUN sudo apt-get install -y sqlite3 libsqlite3-dev python3 python3-pip RUN pip3 install 'pyyaml>5.1' + +COPY . . + +WORKDIR bin + +CMD [ "python3", "gendb.py"] From f626023e3f1cd75f719632692fcd222aa4c80721 Mon Sep 17 00:00:00 2001 From: davidcorne Date: Tue, 7 Jul 2020 14:34:16 +0100 Subject: [PATCH 05/12] Don't pollute the docker root --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d4b8330..a39d5b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,8 @@ RUN sudo apt-get -y upgrade RUN sudo apt-get install -y sqlite3 libsqlite3-dev python3 python3-pip RUN pip3 install 'pyyaml>5.1' -COPY . . +COPY . /project -WORKDIR bin +WORKDIR /project/bin CMD [ "python3", "gendb.py"] From 518d22ba4dd5cd22e8b64a3e3be6f10e6541955f Mon Sep 17 00:00:00 2001 From: David Corne Date: Thu, 16 Jul 2020 11:00:37 +0100 Subject: [PATCH 06/12] Add Docker instructions to README --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 44ef649..c233e1f 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,25 @@ limitations on concurrent writes won't be an issue either. ## Stop yacking and tell me how to generate the database from the YAMl! -Ok! +Ok! There are two ways of doing this; using Docker containers, and manually +setting things up. If you just want to generate the database and use it, I'd +recommend Docker, if you want to actively develop this repo the manual setup +may be preferable. + +### Using Docker + +First you will need docker, you can set that up from [here](https://www.docker.com/get-started). + +The included Dockerfile is set up so that it will install the correct +dependencies and run the database generation script. Everyone's workflow will +differ, but this will get you a generated database: + +``` +docker build -t pathfinder-2-sqlite -f .\Dockerfile . +docker run -it pathfinder-2-sqlite +``` + +### Manual Setup You'll need a working python3 installation and the module `pyyaml` installed. From 0618dae72f4e7c365b54ba7f73c8d9e663d0d524 Mon Sep 17 00:00:00 2001 From: David Corne Date: Thu, 16 Jul 2020 11:28:02 +0100 Subject: [PATCH 07/12] Don't need to run docker interactively --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c233e1f..43b6df6 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ differ, but this will get you a generated database: ``` docker build -t pathfinder-2-sqlite -f .\Dockerfile . -docker run -it pathfinder-2-sqlite +docker run -t pathfinder-2-sqlite ``` ### Manual Setup From 094a0e6ef553e0973d306b43773b0fa2108df059 Mon Sep 17 00:00:00 2001 From: David Corne Date: Thu, 16 Jul 2020 11:37:24 +0100 Subject: [PATCH 08/12] Open all files as binary --- bin/gendb.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/bin/gendb.py b/bin/gendb.py index 1587498..522273a 100644 --- a/bin/gendb.py +++ b/bin/gendb.py @@ -62,96 +62,96 @@ def main(): do_weaponcategory(data['weaponcategory'], conn) # move on to traits - with open('traits.yaml') as yl: + with open('traits.yaml', 'rb') as yl: data = yaml.full_load(yl) do_traits(data, conn) # does both trait types and traits # move on to sources - with open('sources.yaml') as yl: + with open('sources.yaml', 'rb') as yl: data = yaml.full_load(yl) do_sources(data, conn) do_source_entry_table(conn) # move on to senses - with open('senses.yaml') as yl: + with open('senses.yaml', 'rb') as yl: data = yaml.full_load(yl) do_senses(data, conn) # move on to skills - with open('skills.yaml') as yl: + with open('skills.yaml', 'rb') as yl: data = yaml.full_load(yl) do_skills(data, conn) # move on to damagecategory and damagetype - with open('damage.yaml') as yl: + with open('damage.yaml', 'rb') as yl: data = yaml.full_load(yl) do_damage(data, conn) # move on to conditions - with open('conditions.yaml') as yl: + with open('conditions.yaml', 'rb') as yl: data = yaml.full_load(yl) do_conditions(data, conn) # move on to backgrounds - with open('backgrounds.yaml') as yl: + with open('backgrounds.yaml', 'rb') as yl: data = yaml.full_load(yl) do_backgrounds(data, conn) # move on to bulks - with open('bulks.yaml') as yl: + with open('bulks.yaml', 'rb') as yl: data = yaml.full_load(yl) do_bulks(data, conn) # move on to langs - with open('langs.yaml') as yl: + with open('langs.yaml', 'rb') as yl: data = yaml.full_load(yl) do_langs(data, conn) # move on to actions - with open('actions.yaml') as yl: + with open('actions.yaml', 'rb') as yl: data = yaml.full_load(yl) do_actions(data, conn) # move on to spells # TODO do spells once data is proofread - with open('spells.yaml') as yl: + with open('spells.yaml', 'rb') as yl: data = yaml.full_load(yl) do_spells(data, conn) # move on to requirements - with open('requirements.yaml') as yl: + with open('requirements.yaml', 'rb') as yl: data = yaml.full_load(yl) do_requirements(data, conn) # move on to triggers - with open('triggers.yaml') as yl: + with open('triggers.yaml', 'rb') as yl: data = yaml.full_load(yl) do_triggers(data, conn) # move on to armor - with open('armor.yaml') as yl: + with open('armor.yaml', 'rb') as yl: data = yaml.full_load(yl) do_armor(data, conn) # move on to ammo - with open('ammunition.yaml') as yl: + with open('ammunition.yaml', 'rb') as yl: data = yaml.full_load(yl) do_ammo(data, conn) - with open('gear.yaml') as yl: + with open('gear.yaml', 'rb') as yl: data = yaml.full_load(yl) do_gear(data, conn) - with open('feats.yaml') as yl: + with open('feats.yaml', 'rb') as yl: data = yaml.full_load(yl) do_feats(data, conn) - with open('ancestriesheritages.yaml') as yl: + with open('ancestriesheritages.yaml', 'rb') as yl: data = yaml.full_load(yl) do_ancestries(data, conn) # Must be called after feats are loaded - with open('ancestriesheritages.yaml') as yl: + with open('ancestriesheritages.yaml', 'rb') as yl: data = yaml.full_load(yl) do_heritages(data, conn) From de7a7731140d4935fc6bb7fb3e363619d99e4924 Mon Sep 17 00:00:00 2001 From: David Corne Date: Thu, 16 Jul 2020 11:41:39 +0100 Subject: [PATCH 09/12] Set the python encoding to be UTF8 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a39d5b2..76659cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ RUN sudo apt-get -y upgrade RUN sudo apt-get install -y sqlite3 libsqlite3-dev python3 python3-pip RUN pip3 install 'pyyaml>5.1' +ENV PYTHONIOENCODING UTF-8 COPY . /project WORKDIR /project/bin From bd18460d601ab917fcd9ed4ef14ea35637323e7e Mon Sep 17 00:00:00 2001 From: David Corne Date: Thu, 16 Jul 2020 11:42:54 +0100 Subject: [PATCH 10/12] Comment the dockerfile better --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 76659cf..75425f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,18 @@ FROM ubuntu:trusty + +# Install the prerequisites for this RUN sudo apt-get -y update RUN sudo apt-get -y upgrade RUN sudo apt-get install -y sqlite3 libsqlite3-dev python3 python3-pip RUN pip3 install 'pyyaml>5.1' +# Ensure that python runs everything as UTF-8 compatible ENV PYTHONIOENCODING UTF-8 + +# Copy the project into the docker image COPY . /project WORKDIR /project/bin +# Run the database creation script CMD [ "python3", "gendb.py"] From 5d3147df71bcd1136d23a744c778efff4a810e54 Mon Sep 17 00:00:00 2001 From: David Corne Date: Thu, 16 Jul 2020 13:16:45 +0100 Subject: [PATCH 11/12] Update the docker instructions to copy the db out --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 43b6df6..b374302 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,14 @@ may be preferable. First you will need docker, you can set that up from [here](https://www.docker.com/get-started). The included Dockerfile is set up so that it will install the correct -dependencies and run the database generation script. Everyone's workflow will -differ, but this will get you a generated database: +dependencies and run the database generation script. This will get you a +generated database, and copy it into your working directory: ``` docker build -t pathfinder-2-sqlite -f .\Dockerfile . -docker run -t pathfinder-2-sqlite +docker run --name pathfinder-2-sqlite-container -t pathfinder-2-sqlite +docker cp pathfinder-2-sqlite-container:/project/tmp.db . +docker container rm pathfinder-2-sqlite-container ``` ### Manual Setup From 2b221475c7a46ce80567a0b5241cc456b881bf70 Mon Sep 17 00:00:00 2001 From: David Corne Date: Mon, 3 Aug 2020 09:23:17 +0100 Subject: [PATCH 12/12] Don't worry about rb file mode with utf8 encoding --- bin/gendb.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/bin/gendb.py b/bin/gendb.py index e502439..45a70bf 100644 --- a/bin/gendb.py +++ b/bin/gendb.py @@ -49,7 +49,7 @@ def main(): print("{}".format(e)) # Load in the yaml data - with open('basics.yaml', 'rb', encoding='utf-8') as yl: + with open('basics.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) # call the functions to input to SQL do_abilityscore(data['abilityscore'], conn) @@ -62,96 +62,96 @@ def main(): do_weaponcategory(data['weaponcategory'], conn) # move on to traits - with open('traits.yaml', 'rb', encoding='utf-8') as yl: + with open('traits.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_traits(data, conn) # does both trait types and traits # move on to sources - with open('sources.yaml', 'rb', encoding='utf-8') as yl: + with open('sources.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_sources(data, conn) do_source_entry_table(conn) # move on to senses - with open('senses.yaml', 'rb', encoding='utf-8') as yl: + with open('senses.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_senses(data, conn) # move on to skills - with open('skills.yaml', 'rb', encoding='utf-8') as yl: + with open('skills.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_skills(data, conn) # move on to damagecategory and damagetype - with open('damage.yaml', 'rb', encoding='utf-8') as yl: + with open('damage.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_damage(data, conn) # move on to conditions - with open('conditions.yaml', 'rb', encoding='utf-8') as yl: + with open('conditions.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_conditions(data, conn) # move on to backgrounds - with open('backgrounds.yaml', 'rb', encoding='utf-8') as yl: + with open('backgrounds.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_backgrounds(data, conn) # move on to bulks - with open('bulks.yaml', 'rb', encoding='utf-8') as yl: + with open('bulks.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_bulks(data, conn) # move on to langs - with open('langs.yaml', 'rb', encoding='utf-8') as yl: + with open('langs.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_langs(data, conn) # move on to actions - with open('actions.yaml', 'rb', encoding='utf-8') as yl: + with open('actions.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_actions(data, conn) # move on to spells # TODO do spells once data is proofread - with open('spells.yaml', 'rb', encoding='utf-8') as yl: + with open('spells.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_spells(data, conn) # move on to requirements - with open('requirements.yaml', 'rb', encoding='utf-8') as yl: + with open('requirements.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_requirements(data, conn) # move on to triggers - with open('triggers.yaml', 'rb', encoding='utf-8') as yl: + with open('triggers.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_triggers(data, conn) # move on to armor - with open('armor.yaml', 'rb', encoding='utf-8') as yl: + with open('armor.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_armor(data, conn) # move on to ammo - with open('ammunition.yaml', 'rb', encoding='utf-8') as yl: + with open('ammunition.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_ammo(data, conn) - with open('gear.yaml', 'rb', encoding='utf-8') as yl: + with open('gear.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_gear(data, conn) - with open('feats.yaml', 'rb', encoding='utf-8') as yl: + with open('feats.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_feats(data, conn) - with open('ancestriesheritages.yaml', 'rb', encoding='utf-8') as yl: + with open('ancestriesheritages.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_ancestries(data, conn) # Must be called after feats are loaded - with open('ancestriesheritages.yaml', 'rb', encoding='utf-8') as yl: + with open('ancestriesheritages.yaml', encoding='utf-8') as yl: data = yaml.full_load(yl) do_heritages(data, conn)