From fb29ba482d96cf2d41039aff99658d17de5872f9 Mon Sep 17 00:00:00 2001 From: James Miller Date: Thu, 8 Aug 2019 17:28:33 -0500 Subject: [PATCH 01/15] working on spells still --- data/third_party_json/spells.py | 4 ++++ gendb.sh | 2 ++ schema/spells.sql | 1 - 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index aad7eda..169e370 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -1,4 +1,5 @@ import json +import sqlite3 def main(): # load json into python @@ -25,6 +26,9 @@ def main(): sorted_dicts.append(x) # NOW we can go alphabetically spell by spell + + ## Get database connection + conn = sqlite3.connect('../../pf2.db') for i in sorted_dicts: do_sql(i) diff --git a/gendb.sh b/gendb.sh index 9fdfeaf..0205c71 100755 --- a/gendb.sh +++ b/gendb.sh @@ -8,6 +8,7 @@ sqlite3 pf2.db < schema/bulk.sql sqlite3 pf2.db < schema/sizes.sql sqlite3 pf2.db < schema/langs.sql sqlite3 pf2.db < schema/traits.sql +sqlite3 pf2.db < schema/spells.sql sqlite3 pf2.db < schema/feats.sql sqlite3 pf2.db < schema/senses.sql sqlite3 pf2.db < schema/ancestries.sql @@ -22,6 +23,7 @@ sqlite3 pf2.db < data/senses.sql sqlite3 pf2.db < data/sizes.sql sqlite3 pf2.db < data/langs.sql sqlite3 pf2.db < data/traits.sql +sqlite3 pf2.db < data/spells.sql sqlite3 pf2.db < data/feats.sql sqlite3 pf2.db < data/ancestries.sql sqlite3 pf2.db < data/armor.sql diff --git a/schema/spells.sql b/schema/spells.sql index 2ad2a4d..749498c 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -33,7 +33,6 @@ CREATE TABLE spells ( name TEXT NOT NULL UNIQUE, source TEXT, level INTEGER NOT NULL, - has_trigger BOOLEAN NOT NULL, trigger TEXT, descr TEXT NOT NULL, spelltypes_id INTEGER NOT NULL, From 40a1c23c8c5b75aae57bca61867db9679c45bb95 Mon Sep 17 00:00:00 2001 From: James Miller Date: Thu, 8 Aug 2019 17:33:31 -0500 Subject: [PATCH 02/15] fix typos in data/spells.sql --- data/spells.sql | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/data/spells.sql b/data/spells.sql index a90585e..513f98d 100644 --- a/data/spells.sql +++ b/data/spells.sql @@ -1,9 +1,13 @@ + INSERT INTO spelltypes ( + spelltypes_id, + name +) +VALUES (1, 'Spell'), (2, 'Cantrip'), (3, 'Focus'), - (3, 'Ritual') -); + (4, 'Ritual'); INSERT INTO spellcomponents ( spellcomponents_id, @@ -41,4 +45,4 @@ VALUES (5, 1, '298','Evocation', 'TODO'), (6, 1, '298','Illusion', 'TODO'), (7, 1, '298','Necromancy', 'TODO'), - (8, 1, '298','Transmutation', 'Transmutation spells make alterations to or transform the physical form of a creature or object. The morph and polymorph traits appear primarily in transmutation spells.'), + (8, 1, '298','Transmutation', 'Transmutation spells make alterations to or transform the physical form of a creature or object. The morph and polymorph traits appear primarily in transmutation spells.'); From 85f251adc3f6867f60375b7d766a0d5473d7c719 Mon Sep 17 00:00:00 2001 From: James Miller Date: Thu, 8 Aug 2019 18:05:21 -0500 Subject: [PATCH 03/15] still working on spell import --- data/third_party_json/spells.py | 35 ++++++++++++++++++++++++++++++--- schema/spells.sql | 2 +- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index 169e370..3863707 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -29,12 +29,41 @@ def main(): ## Get database connection conn = sqlite3.connect('../../pf2.db') + + id = 0 for i in sorted_dicts: - do_sql(i) + id += 1 + do_sql(i, id, conn) # TODO write this function after sql schema drafted -def do_sql(): - pass +def do_sql(i, id, conn): + print("Doing spell id #{}: {}".format(id, i['name'])) + stmt = """INSERT INTO spells ( + spells_id, + sources_id, + sources_pages, + nethysurl, + name, + source, + level, + descr, + range_text) + VALUES (?,?,?,?,?,?,?,?,?)""" + + rge = None + if 'range' in i: + rge = i['range'] + + dscr = None + if 'description' in i: + dscr = i['description'] + + inp = (id, 1, None, i['nethysUrl'], i['name'], i['source'], i['level'], dscr, rge) + try: + conn.execute(stmt, inp) + except: + print("Error") + conn.commit() if __name__ == "__main__": diff --git a/schema/spells.sql b/schema/spells.sql index 749498c..22bcbb5 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -35,7 +35,7 @@ CREATE TABLE spells ( level INTEGER NOT NULL, trigger TEXT, descr TEXT NOT NULL, - spelltypes_id INTEGER NOT NULL, + spelltypes_id INTEGER, range_text TEXT, range_ft INTEGER, targets TEXT, From d9ffb127e4fc7af1e2348d6aa00ced728b6c0be2 Mon Sep 17 00:00:00 2001 From: James Miller Date: Thu, 8 Aug 2019 20:39:36 -0500 Subject: [PATCH 04/15] spells.py now converts text range data to integer --- data/third_party_json/spells.py | 48 +++++++++++++++++++++++++++------ schema/spells.sql | 10 +++---- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index 3863707..8244200 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -33,10 +33,41 @@ def main(): id = 0 for i in sorted_dicts: id += 1 - do_sql(i, id, conn) + # insert basics of a spell + do_basic_sql(i, id, conn) + do_range_numbers(i,id,conn) + # TODO do all the traits, FK stuff etc... -# TODO write this function after sql schema drafted -def do_sql(i, id, conn): +def do_range_numbers(i, id, conn): + # no need to do range + if 'range' not in i: + return + rg = -1 + # convert range_text to an integer representation + if i['range'] == 'touch': + rg = 0 + elif i['range'] == 'planetary': + rg = 999999999 + # is the only one in CRB with emanation 40' from current scraping + elif i['name'] == 'Repulsion': + rg = 40 + else: + # DO SPLITS + splits = i['range'].split(' ') + # print(splits) + rg = splits[0] + inp = (rg, id) + stmt = "UPDATE spells SET range_ft=? WHERE spells_id=?" + try: + conn.execute(stmt, inp) + except: + print("Error updating range_ft") + else: + conn.commit() + # print("Successfully updated range_ft") + + +def do_basic_sql(i, id, conn): print("Doing spell id #{}: {}".format(id, i['name'])) stmt = """INSERT INTO spells ( spells_id, @@ -44,11 +75,10 @@ def do_sql(i, id, conn): sources_pages, nethysurl, name, - source, level, descr, range_text) - VALUES (?,?,?,?,?,?,?,?,?)""" + VALUES (?,?,?,?,?,?,?,?)""" rge = None if 'range' in i: @@ -58,12 +88,14 @@ def do_sql(i, id, conn): if 'description' in i: dscr = i['description'] - inp = (id, 1, None, i['nethysUrl'], i['name'], i['source'], i['level'], dscr, rge) + inp = (id, 1, i['source'], i['nethysUrl'], i['name'], i['level'], dscr, rge) try: conn.execute(stmt, inp) except: - print("Error") - conn.commit() + print("Error inserting row") + else: + conn.commit() + # print("Successfully inserted row") if __name__ == "__main__": diff --git a/schema/spells.sql b/schema/spells.sql index 22bcbb5..f89d039 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -24,21 +24,21 @@ CREATE TABLE spellschools ( FOREIGN KEY (sources_id) REFERENCES sources(sources_id) ); - +-- TODO eventually once data is finalized, lock down variables as NOT NULL / +-- UNIQUE as sanity requires :) CREATE TABLE spells ( spells_id INTEGER PRIMARY KEY, sources_id INTEGER NOT NULL, sources_pages TEXT, - nethysurl TEXT, name TEXT NOT NULL UNIQUE, - source TEXT, - level INTEGER NOT NULL, + level INTEGER, trigger TEXT, - descr TEXT NOT NULL, + descr TEXT, spelltypes_id INTEGER, range_text TEXT, range_ft INTEGER, targets TEXT, + nethysurl TEXT, FOREIGN KEY (sources_id) REFERENCES sources(sources_id), FOREIGN KEY (spelltypes_id) REFERENCES spelltypes(spelltypes_id) ); From 5f50cd22315706a4d8e13efd990a262571044c9b Mon Sep 17 00:00:00 2001 From: James Miller Date: Thu, 8 Aug 2019 21:01:12 -0500 Subject: [PATCH 05/15] got spells.py fixing the nethys sources_pages field --- data/third_party_json/spells.py | 26 ++++++++++++++++++++++++++ schema/spells.sql | 22 +++++++++++----------- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index 8244200..b8bb021 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -36,8 +36,34 @@ def main(): # insert basics of a spell do_basic_sql(i, id, conn) do_range_numbers(i,id,conn) + do_sources_pages(i,id,conn) # TODO do all the traits, FK stuff etc... +def do_sources_pages(i, id, conn): + if 'source' not in i: + return + print(i) + + res = '' + source_id = 0 + # Do Core Rulebook branch + if "Core Rulebook" in i['source']: + res = i['source'].replace('Core Rulebook pg.','').strip() + source_id = 1 + + stmt = "UPDATE spells SET sources_id=?, sources_pages=? WHERE spells_id=?" + inp = (source_id, res, id) + + try: + conn.execute(stmt, inp) + except: + print("Error updating sources") + else: + conn.commit() + + + + def do_range_numbers(i, id, conn): # no need to do range if 'range' not in i: diff --git a/schema/spells.sql b/schema/spells.sql index f89d039..049ab51 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -28,17 +28,17 @@ CREATE TABLE spellschools ( -- UNIQUE as sanity requires :) CREATE TABLE spells ( spells_id INTEGER PRIMARY KEY, - sources_id INTEGER NOT NULL, - sources_pages TEXT, - name TEXT NOT NULL UNIQUE, - level INTEGER, - trigger TEXT, - descr TEXT, - spelltypes_id INTEGER, - range_text TEXT, - range_ft INTEGER, - targets TEXT, - nethysurl TEXT, + sources_id INTEGER NOT NULL, -- manually entered right now + sources_pages TEXT, -- TODO convert to our format in spells.py + name TEXT NOT NULL UNIQUE, -- scraped from github repo + level INTEGER, -- scraped from github repo + trigger TEXT, -- TODO in spells.py + descr TEXT, -- scraped from github repo + spelltypes_id INTEGER, -- TODO in spells.py + range_text TEXT, -- scraped from github repo + range_ft INTEGER, -- generated from text in spells.py + targets TEXT, -- TODO in spells.py + nethysurl TEXT, -- scraped from github repo FOREIGN KEY (sources_id) REFERENCES sources(sources_id), FOREIGN KEY (spelltypes_id) REFERENCES spelltypes(spelltypes_id) ); From a2690994ae676d291bc943972e9ecbcafc246406 Mon Sep 17 00:00:00 2001 From: James Miller Date: Thu, 8 Aug 2019 21:01:45 -0500 Subject: [PATCH 06/15] remove debug statement --- data/third_party_json/spells.py | 1 - 1 file changed, 1 deletion(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index b8bb021..c9b9224 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -42,7 +42,6 @@ def main(): def do_sources_pages(i, id, conn): if 'source' not in i: return - print(i) res = '' source_id = 0 From e8ab1860d379746fb3e47f9ae5a47f880e7d63b7 Mon Sep 17 00:00:00 2001 From: James Miller Date: Thu, 8 Aug 2019 21:42:57 -0500 Subject: [PATCH 07/15] got spell traits inserting based on github data --- data/third_party_json/spells.py | 40 +++++++++++++++++++++++++++++++++ schema/spells.sql | 5 +++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index c9b9224..a88fd09 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -30,6 +30,15 @@ def main(): ## Get database connection conn = sqlite3.connect('../../pf2.db') + # load in ids for traits from traits table so we only call this once + # instead of every spell + stmt = "SELECT trait_id, short_name FROM traits" + c = conn.cursor() + c.execute(stmt) + traits = c.fetchall() + # print(traits) + + id = 0 for i in sorted_dicts: id += 1 @@ -37,8 +46,39 @@ def main(): do_basic_sql(i, id, conn) do_range_numbers(i,id,conn) do_sources_pages(i,id,conn) + do_spell_traits(i,id,conn,traits) # TODO do all the traits, FK stuff etc... +def do_spell_traits(i, id, conn, traits): + + # get list of traits from the json and capitalize first letter + traits_json = [] + for item in i['traits']: + traits_json.append(item.capitalize()) + + trait_ids =[] + for j in traits_json: + for k in traits: + if j == k[1]: + trait_ids.append(k[0]) + # print(trait_ids) + + inp = [] + for j in trait_ids: + inp.append((id,j)) + # print(inp) + + # insert into sql + stmt = "INSERT OR REPLACE INTO spells_traits (spells_id, traits_id) VALUES (?,?)" + try: + conn.executemany(stmt, inp) + except: + print("Error updating traits") + else: + conn.commit() + + + def do_sources_pages(i, id, conn): if 'source' not in i: return diff --git a/schema/spells.sql b/schema/spells.sql index 049ab51..da18251 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -28,8 +28,8 @@ CREATE TABLE spellschools ( -- UNIQUE as sanity requires :) CREATE TABLE spells ( spells_id INTEGER PRIMARY KEY, - sources_id INTEGER NOT NULL, -- manually entered right now - sources_pages TEXT, -- TODO convert to our format in spells.py + sources_id INTEGER NOT NULL, -- generated in spells.py from scraped data + sources_pages TEXT, -- generated in spells.py from scraped data name TEXT NOT NULL UNIQUE, -- scraped from github repo level INTEGER, -- scraped from github repo trigger TEXT, -- TODO in spells.py @@ -55,6 +55,7 @@ CREATE TABLE spells_traits ( id INTEGER PRIMARY KEY, spells_id INTEGER NOT NULL, traits_id INTEGER NOT NULL, + UNIQUE(spells_id, traits_id), FOREIGN KEY (spells_id) REFERENCES spells(spells_id), FOREIGN KEY (traits_id) REFERENCES traits(traits_id) ); From b0b6a20b83e9591eb9ceec605377cb11f5b47a70 Mon Sep 17 00:00:00 2001 From: James Miller Date: Thu, 8 Aug 2019 21:53:43 -0500 Subject: [PATCH 08/15] spells.py got spell types working --- data/third_party_json/spells.py | 27 +++++++++++++++++++++++++++ schema/spells.sql | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index a88fd09..f17c08b 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -38,6 +38,14 @@ def main(): traits = c.fetchall() # print(traits) + # load in ids for spelltypes from spelltypes table so we only call this once + # instead of every spell + stmt = "SELECT spelltypes_id, name FROM spelltypes" + c = conn.cursor() + c.execute(stmt) + stypes = c.fetchall() + # print(traits) + id = 0 for i in sorted_dicts: @@ -47,8 +55,27 @@ def main(): do_range_numbers(i,id,conn) do_sources_pages(i,id,conn) do_spell_traits(i,id,conn,traits) + do_spell_types(i,id,conn,stypes) # TODO do all the traits, FK stuff etc... +def do_spell_types(i,id,conn,stypes): + res = 0 + for j in stypes: + if i['type'] == j[1]: + res = j[0] + print(id , res) + + inp = (res, id) + + stmt = "UPDATE spells SET spelltypes_id=? WHERE spells_id=?" + + try: + conn.execute(stmt, inp) + except: + print("Error updating spell types") + else: + conn.commit() + def do_spell_traits(i, id, conn, traits): # get list of traits from the json and capitalize first letter diff --git a/schema/spells.sql b/schema/spells.sql index da18251..d2efcda 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -34,7 +34,7 @@ CREATE TABLE spells ( level INTEGER, -- scraped from github repo trigger TEXT, -- TODO in spells.py descr TEXT, -- scraped from github repo - spelltypes_id INTEGER, -- TODO in spells.py + spelltypes_id INTEGER, -- generated from spells.py range_text TEXT, -- scraped from github repo range_ft INTEGER, -- generated from text in spells.py targets TEXT, -- TODO in spells.py From 83f140d3a6e58b27a56eedafa4bad84a26803bb8 Mon Sep 17 00:00:00 2001 From: James Miller Date: Sat, 10 Aug 2019 15:13:24 -0500 Subject: [PATCH 09/15] spells now have triggers working in them --- data/third_party_json/spells.py | 23 ++++++++++++++++++----- schema/spells.sql | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index f17c08b..0649165 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -46,17 +46,25 @@ def main(): stypes = c.fetchall() # print(traits) + # List the various triggers and see if there are any duplicates + # THERE ARE NOT IN THE CRB SO NOT BOTHERING WITH SEPARATE TRIGGERS TABLE YET + ### trigs = [] + ### for i in sorted_dicts: + ### if 'trigger' in i: + ### trigs.append(i['trigger']) + ### print(sorted(trigs)) + ### print(len(trigs)) + ### print(len(set(trigs))) + id = 0 for i in sorted_dicts: id += 1 - # insert basics of a spell do_basic_sql(i, id, conn) do_range_numbers(i,id,conn) do_sources_pages(i,id,conn) do_spell_traits(i,id,conn,traits) do_spell_types(i,id,conn,stypes) - # TODO do all the traits, FK stuff etc... def do_spell_types(i,id,conn,stypes): res = 0 @@ -169,8 +177,9 @@ def do_basic_sql(i, id, conn): name, level, descr, - range_text) - VALUES (?,?,?,?,?,?,?,?)""" + range_text, + trigger) + VALUES (?,?,?,?,?,?,?,?,?)""" rge = None if 'range' in i: @@ -180,7 +189,11 @@ def do_basic_sql(i, id, conn): if 'description' in i: dscr = i['description'] - inp = (id, 1, i['source'], i['nethysUrl'], i['name'], i['level'], dscr, rge) + trg = None + if 'trigger' in i: + trg = i['trigger'] + + inp = (id, 1, i['source'], i['nethysUrl'], i['name'], i['level'], dscr, rge, trg) try: conn.execute(stmt, inp) except: diff --git a/schema/spells.sql b/schema/spells.sql index d2efcda..1fd9042 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -32,7 +32,7 @@ CREATE TABLE spells ( sources_pages TEXT, -- generated in spells.py from scraped data name TEXT NOT NULL UNIQUE, -- scraped from github repo level INTEGER, -- scraped from github repo - trigger TEXT, -- TODO in spells.py + trigger TEXT, -- scraped from spells.py NOTE, there are no duplicate triggers as of CRB, so not bothering with a separate spell triggers table at this time descr TEXT, -- scraped from github repo spelltypes_id INTEGER, -- generated from spells.py range_text TEXT, -- scraped from github repo From a57b6577f88d3a47e239b7c707bc984d621677e6 Mon Sep 17 00:00:00 2001 From: James Miller Date: Sat, 10 Aug 2019 15:27:00 -0500 Subject: [PATCH 10/15] got area working in a flat text field; TODO make area table and refactor --- data/third_party_json/spells.py | 23 +++++++++++++++++++---- schema/spells.sql | 6 +++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index 0649165..af68c44 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -44,7 +44,15 @@ def main(): c = conn.cursor() c.execute(stmt) stypes = c.fetchall() - # print(traits) + + # TODO FIX THIS FOR SPELL COMPONENTS + + # load in ids for spelltypes from spelltypes table so we only call this once + # instead of every spell + stmt = "SELECT spelltypes_id, name FROM spelltypes" + c = conn.cursor() + c.execute(stmt) + stypes = c.fetchall() # List the various triggers and see if there are any duplicates # THERE ARE NOT IN THE CRB SO NOT BOTHERING WITH SEPARATE TRIGGERS TABLE YET @@ -65,6 +73,8 @@ def main(): do_sources_pages(i,id,conn) do_spell_traits(i,id,conn,traits) do_spell_types(i,id,conn,stypes) + # TODO spell components + # TODO spell targets def do_spell_types(i,id,conn,stypes): res = 0 @@ -178,8 +188,9 @@ def do_basic_sql(i, id, conn): level, descr, range_text, - trigger) - VALUES (?,?,?,?,?,?,?,?,?)""" + trigger, + area_text) + VALUES (?,?,?,?,?,?,?,?,?,?)""" rge = None if 'range' in i: @@ -193,7 +204,11 @@ def do_basic_sql(i, id, conn): if 'trigger' in i: trg = i['trigger'] - inp = (id, 1, i['source'], i['nethysUrl'], i['name'], i['level'], dscr, rge, trg) + area = None + if 'area' in i: + area = i['area'] + + inp = (id, 1, i['source'], i['nethysUrl'], i['name'], i['level'], dscr, rge, trg, area) try: conn.execute(stmt, inp) except: diff --git a/schema/spells.sql b/schema/spells.sql index 1fd9042..a27692a 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -26,17 +26,21 @@ CREATE TABLE spellschools ( -- TODO eventually once data is finalized, lock down variables as NOT NULL / -- UNIQUE as sanity requires :) +-- TODO Area eventually needs its own table CREATE TABLE spells ( spells_id INTEGER PRIMARY KEY, sources_id INTEGER NOT NULL, -- generated in spells.py from scraped data sources_pages TEXT, -- generated in spells.py from scraped data name TEXT NOT NULL UNIQUE, -- scraped from github repo level INTEGER, -- scraped from github repo - trigger TEXT, -- scraped from spells.py NOTE, there are no duplicate triggers as of CRB, so not bothering with a separate spell triggers table at this time + trigger TEXT, -- scraped from spells.py NOTE, there are no duplicate triggers + -- as of CRB, so not bothering with a separate spell triggers + -- table at this time descr TEXT, -- scraped from github repo spelltypes_id INTEGER, -- generated from spells.py range_text TEXT, -- scraped from github repo range_ft INTEGER, -- generated from text in spells.py + area_text TEXT, -- TODO need to figure out some sort of programmatic representation for this too targets TEXT, -- TODO in spells.py nethysurl TEXT, -- scraped from github repo FOREIGN KEY (sources_id) REFERENCES sources(sources_id), From cfdef83cb2a33749b5d3ce45c1ab04101900de8f Mon Sep 17 00:00:00 2001 From: James Miller Date: Sat, 10 Aug 2019 15:44:39 -0500 Subject: [PATCH 11/15] spells: have components working too --- data/third_party_json/spells.py | 30 ++++++++++++++++++++++++++---- schema/spells.sql | 1 + 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index af68c44..ce4c8f5 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -46,13 +46,17 @@ def main(): stypes = c.fetchall() # TODO FIX THIS FOR SPELL COMPONENTS + # CREATE TABLE spellcomponents ( + # spellcomponents_id INTEGER PRIMARY KEY, + # name TEXT NOT NULL UNIQUE + # ); # load in ids for spelltypes from spelltypes table so we only call this once # instead of every spell - stmt = "SELECT spelltypes_id, name FROM spelltypes" + stmt = "SELECT spellcomponents_id, name FROM spellcomponents" c = conn.cursor() c.execute(stmt) - stypes = c.fetchall() + ctypes = c.fetchall() # List the various triggers and see if there are any duplicates # THERE ARE NOT IN THE CRB SO NOT BOTHERING WITH SEPARATE TRIGGERS TABLE YET @@ -73,15 +77,33 @@ def main(): do_sources_pages(i,id,conn) do_spell_traits(i,id,conn,traits) do_spell_types(i,id,conn,stypes) - # TODO spell components + do_spell_components(i,id,conn,ctypes) # TODO spell targets +def do_spell_components(i,id,conn,ctypes): + res = None + for j in ctypes: + for k in i['components']: + if k.capitalize() == j[1]: + res = j[0] + + inp = (res, id) + + stmt = "INSERT INTO spells_spellcomponents (spells_id, spellcomponents_id) VALUES (?,?)" + + try: + conn.execute(stmt, inp) + except: + print("Error inserting spell components") + else: + conn.commit() + def do_spell_types(i,id,conn,stypes): res = 0 for j in stypes: if i['type'] == j[1]: res = j[0] - print(id , res) + # print(id , res) inp = (res, id) diff --git a/schema/spells.sql b/schema/spells.sql index a27692a..a1fca2e 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -48,6 +48,7 @@ CREATE TABLE spells ( ); CREATE TABLE spells_spellcomponents( + id INTEGER PRIMARY KEY, spells_id INTEGER NOT NULL, spellcomponents_id INTEGER NOT NULL, FOREIGN KEY (spells_id) REFERENCES spells(spells_id), From b2df5a9ae3718e47cb4f83e2720b6a29cb81c5cc Mon Sep 17 00:00:00 2001 From: James Miller Date: Sat, 10 Aug 2019 15:57:48 -0500 Subject: [PATCH 12/15] spelltargets table populating, but not linked to FK on spells yet --- data/third_party_json/spells.py | 28 ++++++++++++++++++++++++++++ schema/spells.sql | 10 ++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index ce4c8f5..ec665d6 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -68,6 +68,34 @@ def main(): ### print(len(trigs)) ### print(len(set(trigs))) + # List the various targets and see if there are any duplicates + ## YES, there are MANY duplicates, so we need a separate targets table + targs = [] + for i in sorted_dicts: + if 'targets' in i: + targs.append(i['targets']) + dedup_targs = set(targs) + sorted_targs = sorted(dedup_targs) + inp_targs = [] + id = 0 + for i in sorted_targs: + id += 1 + inp_targs.append((id,i)) + stmt = "INSERT INTO spelltargets (spelltargets_id, name) VALUES (?,?)" + try: + conn.executemany(stmt,inp_targs) + except: + print("Error creating targets") + else: + conn.commit() + + + + + # print(sorted(targs)) + # print(len(targs)) + # print(len(set(targs))) + id = 0 for i in sorted_dicts: diff --git a/schema/spells.sql b/schema/spells.sql index a1fca2e..7a5ab90 100644 --- a/schema/spells.sql +++ b/schema/spells.sql @@ -15,6 +15,11 @@ CREATE TABLE spelltraditions ( name TEXT NOT NULL UNIQUE ); +CREATE TABLE spelltargets ( + spelltargets_id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE +); + CREATE TABLE spellschools ( spellschools_id INTEGER PRIMARY KEY, sources_id INTEGER NOT NULL, @@ -41,10 +46,11 @@ CREATE TABLE spells ( range_text TEXT, -- scraped from github repo range_ft INTEGER, -- generated from text in spells.py area_text TEXT, -- TODO need to figure out some sort of programmatic representation for this too - targets TEXT, -- TODO in spells.py + spelltargets_id INTEGER, nethysurl TEXT, -- scraped from github repo FOREIGN KEY (sources_id) REFERENCES sources(sources_id), - FOREIGN KEY (spelltypes_id) REFERENCES spelltypes(spelltypes_id) + FOREIGN KEY (spelltypes_id) REFERENCES spelltypes(spelltypes_id), + FOREIGN KEY (spelltargets_id) REFERENCES spelltargets(spelltargets_id) ); CREATE TABLE spells_spellcomponents( From 6fa96a8013ad5a4487fe2518ee934a78a61fe781 Mon Sep 17 00:00:00 2001 From: James Miller Date: Sat, 10 Aug 2019 16:06:38 -0500 Subject: [PATCH 13/15] spelltargets_id on spells now populating --- data/third_party_json/spells.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index ec665d6..388909e 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -89,6 +89,11 @@ def main(): else: conn.commit() + # load in ids for targets so just doing this once + stmt = "SELECT spelltargets_id, name FROM spelltargets" + c = conn.cursor() + c.execute(stmt) + ttypes = c.fetchall() @@ -106,6 +111,7 @@ def main(): do_spell_traits(i,id,conn,traits) do_spell_types(i,id,conn,stypes) do_spell_components(i,id,conn,ctypes) + do_spell_targets(i,id,conn,ttypes) # TODO spell targets def do_spell_components(i,id,conn,ctypes): @@ -126,6 +132,26 @@ def do_spell_components(i,id,conn,ctypes): else: conn.commit() +def do_spell_targets(i,id,conn,ttypes): + if 'targets' not in i: + return + res = 0 + for j in ttypes: + if i['targets'] == j[1]: + res = j[0] + # print(id , res) + + inp = (res, id) + + stmt = "UPDATE spells SET spelltargets_id=? WHERE spells_id=?" + + try: + conn.execute(stmt, inp) + except: + print("Error updating spelltargets_id") + else: + conn.commit() + def do_spell_types(i,id,conn,stypes): res = 0 for j in stypes: From b0eb30ab25f7d6ef07fab0b1c15ab73fea66b08e Mon Sep 17 00:00:00 2001 From: James Miller Date: Sat, 10 Aug 2019 16:12:03 -0500 Subject: [PATCH 14/15] remove todo on spells.py --- data/third_party_json/spells.py | 1 - 1 file changed, 1 deletion(-) diff --git a/data/third_party_json/spells.py b/data/third_party_json/spells.py index 388909e..9685f18 100644 --- a/data/third_party_json/spells.py +++ b/data/third_party_json/spells.py @@ -112,7 +112,6 @@ def main(): do_spell_types(i,id,conn,stypes) do_spell_components(i,id,conn,ctypes) do_spell_targets(i,id,conn,ttypes) - # TODO spell targets def do_spell_components(i,id,conn,ctypes): res = None From c40afb2c767142086b17b1ad55f80f24d5f94ca5 Mon Sep 17 00:00:00 2001 From: James Miller Date: Sat, 10 Aug 2019 16:17:26 -0500 Subject: [PATCH 15/15] update gendb.sh to generate spell data with spells.py --- gendb.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gendb.sh b/gendb.sh index 0205c71..3b41dbe 100755 --- a/gendb.sh +++ b/gendb.sh @@ -28,5 +28,15 @@ sqlite3 pf2.db < data/feats.sql sqlite3 pf2.db < data/ancestries.sql sqlite3 pf2.db < data/armor.sql sqlite3 pf2.db < data/heritages.sql +# Comment out the following three lines if you don't want to generate the spell data. +cd data/third_party_json +python3 spells.py +cd ../.. + +# TODO Eventually we will stop relying on the spells.py script and I will have +# the actual .sql files for the spell data; I am waiting to see if the +# third-party source improves the data in the next few weeks. If not, we'll +# "divorce" from that data, dump to .sql, and manually manipulate going +# forward.