got skills working with yaml

data/yaml/gendb.py

@@ -6,6 +6,7 @@ import sys
 
 DBFILE = 'tmp.db'
 
+
 def main():
     # delete DBfile and run fresh
     os.remove(DBFILE)
@@ -33,11 +34,143 @@ def main():
     with open('sources.yaml') as yl:
         data = yaml.full_load(yl)
     do_sources(data, conn)
+    do_source_entry_table(conn)
+
+    # move on to skills
+    with open('skills.yaml') as yl:
+        data = yaml.full_load(yl)
+    do_skills(data, conn)
+
+def do_skills(data, conn):
+    # make skill table
+    table = """
+CREATE TABLE skill (
+  skill_id INTEGER PRIMARY KEY,
+  "name" TEXT UNIQUE NOT NULL,
+  descr TEXT
+);
+   """
+    c = conn.cursor()
+    c.execute(table)
+
+    # make sourceentry many-to-many table
+    table = """
+CREATE TABLE sourceentry_skill (
+  id INTEGER PRIMARY KEY,
+  sourceentry_id INTEGER NOT NULL,
+  skill_id INTEGER NOT NULL,
+  UNIQUE (sourceentry_id, skill_id), -- prevent duplicates
+  FOREIGN KEY (sourceentry_id) REFERENCES sourceentry(sourceentry_id),
+  FOREIGN KEY (skill_id) REFERENCES skill(skill_id)
+);
+   """
+    c = conn.cursor()
+    c.execute(table)
+
+    # insert basics into skill table
+    inp_data = []
+    for i in data['skill']:
+        inp_data.append(
+            (i['name'], i['descr']))
+
+    stmt = "INSERT INTO skill (name, descr) VALUES (?,?)"
+    try:
+        conn.executemany(stmt, inp_data)
+    except sqlite3.Error as e:
+        print("Error creating skill: {}".format(e))
+    except:
+        print("Error creating skill something other than sqlite3 error")
+    else:
+        conn.commit()
+
+    # go through and do source entry linking
+
+    for i in data['skill']:
+        print("\n\nDoing the skill: {}".format(i['name']))
+        srcs = []
+        # TODO refactor this inner loop for sources out
+        for j in i['source']:
+            abbr = j['abbr']
+            page_start = j['page_start']
+            if 'page_stop' in j:
+                page_stop = j['page_stop']
+            else:
+                page_stop = page_start
+            srcs.append([i['name'], abbr, page_start, page_stop])
+        print("srcs: {}".format(srcs))
+        do_sourceentry_to_skill(srcs, conn)
+
+
+# TODO ugggh;;; this is soooo ugly and needs refactoring but it's working
+def do_sourceentry_to_skill(srcs, conn):
+    c = conn.cursor()
+
+    stmt = "SELECT source.source_id, skill.skill_id FROM source, skill WHERE source.abbr=? AND skill.name=?"
+    istmt = "INSERT INTO sourceentry (source_id, page_start, page_stop) VALUES (?,?,?)"
+    for i in srcs:
+        print("i in srcs: {}".format(i))
+        inp_data=(i[1],i[0])
+        print("inp data: {}".format(inp_data)) 
+        for row in c.execute(stmt, inp_data):
+            print("source_id:{} skill_id:{}".format(row[0], row[1]))
+            iinp_data = (row[0], i[2], i[3])
+            print("iinp data: {}".format(iinp_data))
+
+            try:
+                c.execute(istmt, iinp_data)
+            except sqlite3.IntegrityError as e:
+                if "UNIQUE" in str(e):
+                    # we fully expect UNIQUE constraint to fail on some of these so it's fine
+                    conn.commit()
+                    print("committed istmt")
+                else:
+                    # but we still want to know what's going on if there's some other error
+                    print("Something went wrong with istmt: {}".format(e))
+            except sqlite3.Error as e:
+                print("Error inserting a sourceentry for skill: {}".format(e))
+            else:
+                conn.commit()
+                print("committed istmt")
+
+            linkstmt = "INSERT INTO sourceentry_skill (sourceentry_id, skill_id) VALUES ((SELECT sourceentry_id from sourceentry WHERE source_id=? AND page_start=? AND page_stop=?), ?)"
+            linkinp_data = (row[0], i[2], i[3], row[1])
+            print(linkinp_data)
+            try:
+                c.execute(linkstmt, linkinp_data)
+            except sqlite3.IntegrityError as e:
+                if "UNIQUE" in str(e):
+                    # we fully expect UNIQUE constraint to fail on some of these so it's fine
+                    conn.commit()
+                    print("committed linkstmt")
+                    pass
+                else:
+                    # but we still want to know what's going on if there's some other error
+                    print(e)
+            except sqlite3.Error as e:
+                print("Error inserting a sourceentry for skill: {}".format(e))
+            else:
+                conn.commit()
+                print("committed linkstmt")
+
+
+def do_source_entry_table(conn):
+    table = """
+CREATE TABLE sourceentry (
+  sourceentry_id INTEGER PRIMARY KEY,
+  source_id INTEGER NOT NULL,
+  page_start INTEGER NOT NULL,
+  page_stop INTEGER NOT NULL,
+  UNIQUE (source_id, page_start, page_stop), -- prevent duplicates
+  FOREIGN KEY (source_id) REFERENCES source(source_id)
+);
+   """
+    c = conn.cursor()
+    c.execute(table)
 
 def do_sources(data, conn):
     table = """
-CREATE TABLE sources (
+CREATE TABLE source (
-  sources_id INTEGER PRIMARY KEY,
+  source_id INTEGER PRIMARY KEY,
   isbn TEXT,
   pzocode TEXT,
   full_name TEXT NOT NULL UNIQUE,
@@ -55,26 +188,22 @@ CREATE TABLE sources (
 
     inp_data = []
     for i in data['source']:
-      inp_data.append((i['isbn'],
+        inp_data.append(
-                       i['pzocode'],
+            (i['isbn'], i['pzocode'], i['full_name'], i['short_name'],
-                       i['full_name'],
+             i['abbr'], i['descr'], i['release_date'], i['is_first_party'],
-                       i['short_name'],
-                       i['abbr'],
-                       i['descr'],
-                       i['release_date'],
-                       i['is_first_party'],
              i['ogl_copyright_block']))
 
-   stmt = "INSERT INTO sources (isbn, pzocode, full_name, short_name, abbr, descr, release_date, is_first_party, ogl_copyright_block) VALUES (?,?,?,?,?,?,?,?,?)"
+    stmt = "INSERT INTO source (isbn, pzocode, full_name, short_name, abbr, descr, release_date, is_first_party, ogl_copyright_block) VALUES (?,?,?,?,?,?,?,?,?)"
     try:
         conn.executemany(stmt, inp_data)
     except sqlite3.Error as e:
-      print("Error creating sources: {}".format(e))
+        print("Error creating source: {}".format(e))
     except:
         print("Error creating sources something other than sqlite3 error")
     else:
         conn.commit()
 
+
 def do_traits(data, conn):
     # create the two tables
     table = """
@@ -111,7 +240,9 @@ CREATE TABLE trait (
     # insert data into trait
     inp_data = []
     for i in data['trait']:
-      inp_data.append((i['descr'], i['name'], i['type'])) # trailing comma necessary for one-item tuple
+        inp_data.append(
+            (i['descr'], i['name'],
+             i['type']))  # trailing comma necessary for one-item tuple
     stmt = "INSERT INTO trait (descr, short_name, traittype_id) SELECT ?,?, traittype_id FROM traittype WHERE traittype.name=?"
     try:
         conn.executemany(stmt, inp_data)
@@ -124,6 +255,7 @@ CREATE TABLE trait (
     else:
         conn.commit()
 
+
 def do_size(data, conn):
     table = """
 CREATE TABLE size (
@@ -140,7 +272,8 @@ CREATE TABLE size (
 
     inp_data = []
     for i in data:
-      inp_data.append((i['name'], i['space_in_ft'], i['reach_tall_ft'], i['reach_long_ft']))
+        inp_data.append((i['name'], i['space_in_ft'], i['reach_tall_ft'],
+                         i['reach_long_ft']))
 
     stmt = "INSERT INTO size (short_name, space_in_ft, reach_tall_ft, reach_long_ft) VALUES (?,?,?,?)"
     try:
@@ -150,6 +283,7 @@ CREATE TABLE size (
     else:
         conn.commit()
 
+
 def do_weaponcategory(data, conn):
     table = """
 CREATE TABLE weaponcategory (
@@ -175,6 +309,7 @@ CREATE TABLE weaponcategory (
     else:
         conn.commit()
 
+
 def do_movement(data, conn):
     table = """
 CREATE TABLE movement (
@@ -200,6 +335,7 @@ CREATE TABLE movement (
     else:
         conn.commit()
 
+
 def do_frequency(data, conn):
     table = """
 CREATE TABLE frequency (
@@ -225,6 +361,7 @@ CREATE TABLE frequency (
     else:
         conn.commit()
 
+
 def do_alignment(data, conn):
     # print(data)
     table = """
@@ -250,6 +387,7 @@ CREATE TABLE alignment (
     else:
         conn.commit()
 
+
 def do_langrarity(data, conn):
     table = """
 CREATE TABLE langrarity (
@@ -275,6 +413,7 @@ CREATE TABLE langrarity (
     else:
         conn.commit()
 
+
 def do_actioncost(data, conn):
     table = """
 CREATE TABLE actioncost (
@@ -299,6 +438,7 @@ CREATE TABLE actioncost (
     else:
         conn.commit()
 
+
 def do_abilityscore(data, conn):
     table = """
 CREATE TABLE abilityscore (
@@ -324,6 +464,7 @@ CREATE TABLE abilityscore (
     else:
         conn.commit()
 
+
 def get_db_conn():
     ## Get database connection
     conn = sqlite3.connect(DBFILE)  # eventually hook this up to be the main db

data/yaml/skills.yaml

@@ -0,0 +1,87 @@
+# TODO all the descriptions
+skill:
+  - name: Acrobatics
+    source:
+    - abbr: CRB
+      page_start: 240
+    descr: TODO
+  - name: Arcana
+    source:
+    - abbr: CRB
+      page_start: 241
+    descr: TODO
+  - name: Athletics
+    source:
+    - abbr: CRB
+      page_start: 241
+    descr: TODO
+  - name: Crafting
+    source:
+    - abbr: CRB
+      page_start: 243
+    descr: TODO
+  - name: Deception
+    source:
+    - abbr: CRB
+      page_start: 245
+    descr: TODO
+  - name: Diplomacy
+    source:
+    - abbr: CRB
+      page_start: 246
+    descr: TODO
+  - name: Intimidation
+    source:
+    - abbr: CRB
+      page_start: 247
+    descr: TODO
+  - name: Lore
+    source:
+    - abbr: CRB
+      page_start: 247
+    descr: TODO
+  - name: Medicine
+    source:
+    - abbr: CRB
+      page_start: 248
+    descr: TODO
+  - name: Nature
+    source:
+    - abbr: CRB
+      page_start: 249
+    descr: TODO
+  - name: Occultism
+    source:
+    - abbr: CRB
+      page_start: 249
+    descr: TODO
+  - name: Performance
+    source:
+    - abbr: CRB
+      page_start: 250
+    descr: TODO
+  - name: Religion
+    source:
+    - abbr: CRB
+      page_start: 250
+    descr: TODO
+  - name: Society
+    source:
+    - abbr: CRB
+      page_start: 250
+    descr: TODO
+  - name: Stealth
+    source:
+    - abbr: CRB
+      page_start: 251
+    descr: TODO
+  - name: Survival
+    source:
+    - abbr: CRB
+      page_start: 252
+    descr: TODO
+  - name: Thievery
+    source:
+    - abbr: CRB
+      page_start: 253
+    descr: TODO