.

.gitignore

@@ -0,0 +1 @@
+.DS_Store

wahapediaImport

@@ -1 +0,0 @@
-Subproject commit 880e89ed6eb9bda2b1c386458463e795627ea91a

wahapedia_import/.gitignore

@@ -0,0 +1,138 @@
+# This script specific
+
+config.py
+wahapedia/
+manifests/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Dependency directories
+node_modules/

wahapedia_import/README.md

@@ -0,0 +1,17 @@
+# wh40kdata
+
+## Rough steps
+
+Install `mysql 5.7`, if not already installed, make a database called `wahapedia`
+
+Create `config.py` based on `example.config.py` and populate with your credentials
+
+Install `python 3` and run `python3 updateInputData.py`
+
+Run `python3 ingestion.py` to populate your local db
+
+Install node, then navigate to `cd exportion` and run `npm i` to install node modules
+
+From `exportion/`, run `npm start` to start the server, then navigate to [http://localhost:3000/](http://localhost:3000/)
+
+Click a faction and then, in the browser’s network inspector, copy the returned object to paste into a manifest’s (im/ex)port window

wahapedia_import/example.config.py

@@ -0,0 +1,4 @@
+host = "localhost"
+user = "[user]"
+passwd = "[userPass]"
+database = "wahapedia"

wahapedia_import/exportion/app.js

@@ -0,0 +1,41 @@
+var createError = require('http-errors');
+var express = require('express');
+var path = require('path');
+const pino = require('express-pino-logger')();
+var cookieParser = require('cookie-parser');
+var logger = require('morgan');
+
+// var indexRouter = require('./routes/index');
+
+var app = express();
+
+// view engine setup
+app.set('views', path.join(__dirname, 'views'));
+app.set('view engine', 'jade');
+
+app.use(logger('dev'));
+app.use(express.json());
+app.use(express.urlencoded({ extended: false }));
+app.use(cookieParser());
+app.use(express.static(path.join(__dirname, 'public')));
+app.use(pino);
+
+app.use('/', require('./routes/index'));
+
+// catch 404 and forward to error handler
+app.use(function(req, res, next) {
+  next(createError(404));
+});
+
+// error handler
+app.use(function(err, req, res, next) {
+  // set locals, only providing error in development
+  res.locals.message = err.message;
+  res.locals.error = req.app.get('env') === 'development' ? err : {};
+
+  // render the error page
+  res.status(err.status || 500);
+  res.render('error');
+});
+
+module.exports = app;

wahapedia_import/exportion/bin/www

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+/**
+ * Module dependencies.
+ */
+
+var app = require('../app');
+var debug = require('debug')('exportion:server');
+var http = require('http');
+
+/**
+ * Get port from environment and store in Express.
+ */
+
+var port = normalizePort(process.env.PORT || '3000');
+app.set('port', port);
+
+/**
+ * Create HTTP server.
+ */
+
+var server = http.createServer(app);
+
+/**
+ * Listen on provided port, on all network interfaces.
+ */
+
+server.listen(port);
+server.on('error', onError);
+server.on('listening', onListening);
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+
+function normalizePort(val) {
+  var port = parseInt(val, 10);
+
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+
+function onError(error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  var bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+  var addr = server.address();
+  var bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  debug('Listening on ' + bind);
+}

wahapedia_import/exportion/config/data.js

@@ -0,0 +1,13 @@
+const mysql = require("mysql");
+
+//Database connection
+const pool  = mysql.createPool({
+  connectionLimit : 7,
+  host            : 'localhost',
+  user            : 'root',
+  password        : 'root',
+  database        : 'wahapedia',
+  charset         : "utf8mb4_unicode_ci"
+});
+
+module.exports = pool;

wahapedia_import/exportion/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "exportion",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "start": "nodemon ./bin/www"
+  },
+  "dependencies": {
+    "body-parser": "^1.19.0",
+    "cookie-parser": "~1.4.4",
+    "debug": "~2.6.9",
+    "express": "~4.16.1",
+    "express-pino-logger": "^4.0.0",
+    "http-errors": "~1.6.3",
+    "jade": "~1.11.0",
+    "morgan": "~1.9.1",
+    "mysql": "^2.18.1",
+    "nodemon": "^1.19.4",
+    "pino-colada": "^1.6.0",
+    "sqlstring": "^2.3.1",
+    "string-similarity": "^4.0.4"
+  }
+}

wahapedia_import/exportion/public/javascripts/manifest.process.js

@@ -0,0 +1,794 @@
+// TODO: Error reporting
+// TODO: Fix incrementation guess
+
+const subFacNames = {
+  AS: 'Order',
+  AdM: 'Forge World',
+  AE: 'Craftworld',
+  AM: 'Regiment',
+  CD: 'Allegiance',
+  QT: 'Dread Household', // Questor Traitoris
+  CSM: 'Legion',
+  DG: 'Plague Company',
+  DRU: 'Kabal', // Wych Cult, Haemunculous Coven
+  GC: 'Cult',
+  GK: 'Brotherhood',
+  QI: 'Noble Household', // Questor Allegiance
+  NEC: 'Dynasty',
+  ORK: 'Clan',
+  SM: 'Chapter',
+  TAU: 'Sept',
+  TS: 'Great Cult',
+  TYR: 'Hive Fleet',
+}
+processInfo = (data,factionKey) => {
+  let factionName = data.factions.find(faction => faction.faction_id == factionKey).name
+  let info = {
+    name: factionName,
+    game: 'Warhammer 40,000',
+    genre: 'sci-fi',
+    publisher: 'GW',
+    url: 'https://warhammer40000.com/',
+    notes: 'This manifest is provided for the purposes of testing the features of *Rosterizer* and is not intended for distribution.',
+    revision: '0.0.1',
+    dependencies: [
+      {
+        slug: "123456",
+        name: "40k9e",
+        game: "Warhammer 40,000"
+      }
+    ],
+    manifest: {},
+  }
+  return info
+}
+processItems = (data) => {
+  let assetCatalog = {'Roster§Army': {}};
+  processModels(data,assetCatalog);
+  processAbilities(data,assetCatalog);
+  processWargear(data,assetCatalog);
+  processPsychicPowers(data,assetCatalog);
+  processWarlordTraits(data,assetCatalog);
+  processUnits(data,assetCatalog);
+  return assetCatalog
+}
+processClasses = data => {
+  let assetTaxonomy = {};
+  processFactions(data,assetTaxonomy);
+  processPsychicClasses(data,assetTaxonomy);
+  return assetTaxonomy
+}
+processModels = (data,assetCatalog) => {
+  data.models.forEach((model,i,a) => {
+    a[i].duplicated = data.models.filter(dataModel => dataModel.name === a[i].name).length - 1
+    a[i].itemKey = 'Model§' + a[i].name;
+    if(a[i].duplicated){
+      let unitName = data.datasheets.filter(datasheet => datasheet.datasheet_id == model.datasheet_id)[0].name;
+      a[i].itemKey += ` (${unitName})`;
+    }
+  });
+  let modelList = Array.from(new Set(data.models.map(model => model.itemKey)));
+  modelList.forEach(modelItemKey => {
+    let dupModels = data.models.filter(model => model.itemKey === modelItemKey);
+    let tempItem = dedupModels(dupModels);
+    if(dupModels[0].duplicated){
+      tempItem.rules = {
+        'rename me': {
+          evals:[],
+          failState: 'pass',
+          evaluate: 'AND',
+          actions: [
+            {
+              paths: [
+                [
+                  '{self}',
+                  'designation'
+                ]
+              ],
+              actionType: 'set',
+              value: dupModels[0].name,
+              iterations: 1
+            }
+          ]
+        }
+      }
+    }
+    assetCatalog[modelItemKey] = tempItem;
+    // console.log(modelItemKey)
+    let tempStatline = JSON.parse(JSON.stringify(tempItem));
+    delete tempStatline.stats.Points;
+    assetCatalog[modelItemKey.replace('Model§','Statline§')] = tempStatline;
+  });
+}
+dedupModels = (dupModels) => {
+  let deduped = dupModels[0];
+  let props = ['attacks','ballistic_skill','base_size','cost','leadership','movement','save','strength','toughness','weapon_skill','wounds'];
+  props.forEach(prop => {
+    let arr = dupModels.map(model => model[prop]);
+    deduped[prop] = findMode(arr);
+  });
+  return {
+    stats: {
+      Points: {value: deduped.cost},
+      M: {value: deduped.movement},
+      WS: {value: deduped.weapon_skill},
+      BS: {value: deduped.ballistic_skill},
+      S: {value: deduped.strength},
+      T: {value: deduped.toughness},
+      W: {value: deduped.wounds},
+      A: {value: deduped.attacks},
+      Ld: {value: deduped.leadership},
+      Sv: {value: deduped.save},
+      Base: {value: deduped.base_size},
+    }
+  }
+}
+findMode = (arr) => {
+  if (arr.length == 0) return null;
+
+  var modeMap = {},
+    maxEl = [arr[0]],
+    maxCount = 1;
+
+  for (var i = 0; i < arr.length; i++) {
+    var el = arr[i];
+
+    if (modeMap[el] == null) modeMap[el] = 1;
+    else modeMap[el]++;
+
+    if (modeMap[el] > maxCount) {
+      maxEl = [el];
+      maxCount = modeMap[el];
+    } else if (modeMap[el] == maxCount) {
+      maxEl.push(el);
+      maxCount = modeMap[el];
+    }
+  }
+  // console.log(maxEl)
+  let val = maxEl.sort((a,b) => {
+    if(typeof a === 'string' && typeof b === 'string') return b.localeCompare(a);
+    else if(typeof a === 'number' && typeof b === 'number') return a-b
+  })[0];
+  return val
+}
+ordinalize = (n,keepNumber = true) => {
+  const ordinals = ['th','st','nd','rd'];
+  let v = n % 100;
+  return (keepNumber?n:'') + (ordinals[(v-20)%10]||ordinals[v]||ordinals[0]);
+}
+processAbilities = (data,assetCatalog) => {
+  data.abilities.abilities.forEach((ability,i,a) => {
+    // console.log(ability.name)
+    // console.log(formatText(ability.description))
+    let shouldLog = false;// = ability.name === 'Tactical Precision (Aura)';
+    let tempAbility = {text: formatText(ability.description,shouldLog)};
+    let itemKey;
+    if(!ability.is_other_wargear){
+      itemKey = 'Ability§' + ability.name;
+      if(!assetCatalog[itemKey]) assetCatalog[itemKey] = tempAbility;
+      else assetCatalog[itemKey].text += '\n\nERROR: The following text was found on another ability with the same name.\n\n' + formatText(ability.description,shouldLog);
+    }else{
+      itemKey = 'Wargear§' + ability.name;
+      let abilityCostArr = data.abilities.datasheets_abilities.filter(datasheets_ability => datasheets_ability.ability_id === ability.ability_id).map(datasheets_ability => datasheets_ability.cost);
+      let costMode = findMode(abilityCostArr);
+      if(costMode) tempAbility.stats = {Points: {value: costMode}};
+      if(!assetCatalog[itemKey]) assetCatalog[itemKey] = tempAbility;
+      else assetCatalog[itemKey].text += '\n\nERROR: The following text was found on another wargear with the same name.\n\n' + formatText(ability.description,shouldLog);
+    }
+    let subFactTest = new RegExp(`&lt;${data.factCurrent}&gt;`, 'gi');
+    if(subFactTest.test(assetCatalog[itemKey].text)){
+      let ruleText = assetCatalog[itemKey].text.replace(subFactTest,'{v}')
+      assetCatalog[itemKey].text = '';
+      // console.log(ruleText)
+      assetCatalog[itemKey].stats = assetCatalog[itemKey].stats || {};
+      assetCatalog[itemKey].stats[data.factCurrent] = {
+        statType: 'term',
+        value: `&lt;${data.factCurrent.toUpperCase()}&gt;`,
+        text: ruleText,
+        visibility: 'hidden',
+        dynamic: true
+      }
+    }
+    a[i].itemKey = itemKey;
+    data.text = data.text || '';
+    data.text += assetCatalog[itemKey].text;
+  });
+  data.abilities.composed = [];
+  data.abilities.datasheets_abilities.forEach((element) => {
+    data.abilities.composed.push({
+      ...element,
+      ...data.abilities.abilities.find(e=>e.ability_id===element.ability_id)
+    });  
+  });
+}
+processWargear = (data,assetCatalog) => {
+  data.wargear.wargear_list.forEach(wargear => {
+    let weapName = wargear.name.replace(/(1: |2: |3: )/,'').replace(/в/g,'d');
+    let tempWeapon = {stats:{
+      AP: {value: wargear.armor_piercing},
+      D: {value: wargear.damage},
+      S: {value: wargear.strength},
+      Type: {value: wargear.type},
+      Range: {value: wargear.weapon_range},
+    }};
+    if(wargear.abilities) tempWeapon.text = formatText(wargear.abilities);
+    let wargearArr = data.wargear.wargear_list.filter(wargear_list => wargear_list.wargear_id == wargear.wargear_id).map(wargear => 'Weapon§' + wargear.name);
+    let costArr = data.wargear.datasheets_wargear.filter(datasheets_wargear => datasheets_wargear.wargear_id == wargear.wargear_id).map(wargear => wargear.cost);
+    let cost = findMode(costArr);
+    // console.log(wargear.wargear_id,cost,costArr)
+    if(wargearArr.length == 1 && cost){
+      tempWeapon.stats.Points = {value: cost};
+    }
+    assetCatalog['Weapon§' + weapName] = tempWeapon;
+    if(wargearArr.length > 1){
+      // console.log(wargearArr,weapName)
+      let itemKey = 'Wargear§' + data.wargear.wargear.filter(gear => gear.wargear_id == wargear.wargear_id)[0].name;
+      assetCatalog[itemKey] = assetCatalog[itemKey] || {};
+      assetCatalog[itemKey].assets = assetCatalog[itemKey].assets || {};
+      assetCatalog[itemKey].assets.traits = assetCatalog[itemKey].assets.traits || [];
+      assetCatalog[itemKey].assets.traits.push('Weapon§' + weapName);
+      if(cost){
+        assetCatalog[itemKey].stats = {Points: {value: cost}};
+      }
+    }
+  });
+  let shootingMelee = data.wargear.wargear_list.filter(wargear => wargear.name.includes('(shooting)') || wargear.name.includes('(melee)'));
+  let shooting = shootingMelee.filter(wargear => wargear.name.includes('(shooting)'));
+  let melee = shootingMelee.filter(wargear => wargear.name.includes('(melee)'));
+  shooting.forEach(shooter => {
+    let bareName = shooter.name.replace(' (shooting)','');
+    if(melee.filter(meleer => meleer.name.includes(bareName))){
+      assetCatalog['Wargear§'+bareName] = {
+        assets: {traits:[
+          'Weapon§'+bareName+' (melee)',
+          'Weapon§'+bareName+' (shooting)',
+        ]}
+      }
+      if(assetCatalog['Weapon§'+shooter.name].stats?.Points?.value){
+        assetCatalog['Wargear§'+bareName].stats = assetCatalog['Wargear§'+bareName].stats || {};
+        assetCatalog['Wargear§'+bareName].stats.Points = assetCatalog['Wargear§'+bareName].stats.Points || {};
+        assetCatalog['Wargear§'+bareName].stats.Points.value = assetCatalog['Weapon§'+shooter.name].stats.Points.value;
+        delete assetCatalog['Weapon§'+shooter.name].stats;
+        delete assetCatalog['Weapon§'+bareName+' (melee)'].stats;
+      }
+    }
+  })
+  data.wargear.composed = [];
+  data.wargear.datasheets_wargear.forEach((wargear) => {
+    let wargearArr = data.wargear.wargear_list.filter(wargear_list => wargear_list.wargear_id == wargear.wargear_id);
+    let thisWargear = data.wargear.wargear.find(e=>e.wargear_id===wargear.wargear_id);
+    data.wargear.composed.push({
+      ...wargear,
+      ...thisWargear,
+      itemKey: (wargearArr.length == 1 ? 'Weapon§' : 'Wargear§') + thisWargear.name
+    });  
+  });
+}
+createWargearStat = (i,wargearArr,modelLoadout,assetCatalog) => {
+  var stringSimilarity = require('string-similarity');
+  let bestMatchIndex = stringSimilarity.findBestMatch(modelLoadout[i]?.toLowerCase() || '',wargearArr.map(gear => gear.itemKey.toLowerCase())).bestMatchIndex;
+  let current = i < modelLoadout.length ? wargearArr[bestMatchIndex].itemKey.split('§')[1] : '-';
+  let tempStat = {
+    value: current,
+    label: 'Loadout',
+    statType: 'rank',
+    statOrder: i+1,
+    group: 'Loadout',
+    groupOrder: 2,
+    ranks: {
+      '-': {order: 0},
+    },
+    visibility: 'active',
+    dynamic: true
+  }
+  wargearArr.forEach((wargear,i) => {
+    let wargearName = wargear.itemKey.split('§')[1];
+    let actualTrait = assetCatalog[wargear.itemKey];
+    let assignedTrait = (actualTrait.stats?.Points?.value === undefined && !wargear.cost) || actualTrait.stats?.Points?.value == wargear.cost ? wargear.itemKey : {
+      item: wargear.itemKey,
+      stats: {Points: {value: Number(wargear.cost)}}
+    }
+    tempStat.ranks[wargearName] = {
+      order: i+1,
+      traits: [{trait: assignedTrait}],
+    }
+  });
+  return tempStat
+}
+processPsychicPowers = (data,assetCatalog) => {
+  data.psychicPowers.forEach(power => {
+    if(power.type){
+      let powerName = power.type + '§' + power.name.toLowerCase().split(' ').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
+      let tempPower = {
+        text: formatText(power.description)
+      };
+      if(power.roll) tempPower.stats = {Roll:{value: power.roll}};
+      assetCatalog[powerName] = tempPower;
+    }else{
+      let powerName = 'Psychic Power§' + power.name.toLowerCase().split(' ').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
+      let tempPower = {
+        text: formatText(power.description),
+        stats:{Roll:{
+          value: power.type + (power.roll ? (' ' + power.roll) : '')
+        }}
+      };
+      assetCatalog[powerName] = tempPower;
+    }
+  });
+}
+processWarlordTraits = (data,assetCatalog) => {
+  data.warlordTraits.forEach(trait => {
+    let traitName = 'Warlord Trait§' + trait.name.toLowerCase().split(' ').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
+    let tempTrait = {
+      text: formatText(trait.description),
+      stats:{Discipline:{
+        value: trait.type + (trait.roll ? (' ' + trait.roll) : '')
+      }}
+    };
+    assetCatalog[traitName] = tempTrait;
+  });
+}
+formatText = (text,log = false) => {
+  let newText = text
+  let replacePatterns1 = {
+    pPattern: [/<p[^>]+>((?:.(?!\<\/p\>))*.)<\/p>/g,'$1'],
+    divPattern: [/<div[^>]+>((?:.(?!\<\/div\>))*.)<\/div>/g,'$1'],
+    anchorPattern: [/<a[\s]+href="([^>]+)">((?:.(?!\<\/a\>))*.)<\/a>/g,'[$2](https://wahapedia.ru$1)'],
+    tooltipPattern: [/<span[\s]+class="tooltip([^>]+)>((?:.(?!\<\/span\>))*.)<\/span>/g,'$2'],
+    kwb3Pattern: [/<span[\s]+class="kwb3">((?:.(?!\<\/span\>))*.)<\/span>/g,'$1'],
+    boldunderPattern: [/<span[\s]+class="kwb kwbu">((?:.(?!\<\/span\>))*.)<\/span>/g,'<b><i>$1</i></b>'],
+    boldPattern: [/<span[\s]+class="kwb">((?:.(?!\<\/span\>))*.)<\/span>/g,'<b>$1</b>'],
+    underPattern: [/<span[\s]+class="kwbu">((?:.(?!\<\/span\>))*.)<\/span>/g,'<i>$1<i>'],
+    ttPattern: [/<span[\s]+class="tt">((?:.(?!\<\/span\>))*.)<\/span>/g,'<b>$1</b>'],
+    boldunderPattern2: [/<span[\s]+class="kwb kwbu">((?:.(?!\<\/span\>))*.)<\/span>/g,'<b><i>$1</i></b>'],
+    boldPattern2: [/<span[\s]+class="kwb">((?:.(?!\<\/span\>))*.)<\/span>/g,'<b>$1</b>'],
+    underPattern2: [/<span[\s]+class="kwbu">((?:.(?!\<\/span\>))*.)<\/span>/g,'<i>$1<i>'],
+    ttPattern2: [/<span[\s]+class="tt">((?:.(?!\<\/span\>))*.)<\/span>/g,'<b>$1</b>'],
+    h_customPattern: [/<span[\s]+class="h_custom">((?:.(?!\<\/span\>))*.)<\/span>/g,'<b>$1</b>'],
+    redfontPattern: [/<span[\s]+class="redfont">((?:.(?!\<\/span\>))*.)<\/span>/g,'<b>$1</b>'],
+  }
+  let replacePatterns2 = {
+    doubleBoldEndPattern: [/<\/b><\/b>/g,'</b>'],
+    doubleItalicsEndPattern: [/<\/i>((?:.(?!\<i\>))*)<\/i>/g,'$1<i>'],
+    whitespacePattern1: [/\<\/b\>[\s]\<b\>/g,' '],
+    whitespacePattern2: [/\<\/i\>[\s]\<i\>/g,' '],
+    boldTranslationPattern: [/<\/?b>/g,'**'],
+    doubleBoldPattern: [/\*\*\*\*/g,''],
+    doubleBoldPattern2: [/\*\*\s\*\*/g,' '],
+    italicsTranslationPattern: [/<\/?i>/g,'*']
+  }
+  newText = text.replace(/kwb2/g,'kwb');
+  if(log) console.log(newText)
+  Object.entries(replacePatterns1).forEach(([name,pattern]) => {
+    newText = newText.replace(pattern[0],pattern[1]);
+    if(log) console.log(name,newText)
+  });
+  if(log) console.log(newText)
+  let newTextArr = newText.split('<b>');
+  if(log) console.log(newTextArr)
+  newTextArr.forEach((sliver,i,a) => {
+    if(i > 0 && a[i+1]?.includes('</b>')){
+      a[i+1] = a[i] + a[i+1];
+      delete a[i];
+    }
+  });
+  newText = Object.values(newTextArr).join('<b>');
+  newTextArr = newText.split('<i>');
+  newTextArr.forEach((sliver,i,a) => {
+    if(i > 0 && a[i+1]?.includes('</i>')){
+      a[i+1] = a[i] + a[i+1];
+      delete a[i];
+    }
+  });
+  newText = Object.values(newTextArr).join('<i>');
+  Object.entries(replacePatterns2).forEach(([name,pattern]) => {
+    newText = newText.replace(pattern[0],pattern[1]);
+    if(log) console.log(name,newText)
+  });
+  // newText = newText.replace(/"/g,'″'); // too many html structures get screwed by this
+  newText = newText.replace(/<ul[^>]+><li>/g,'* ');
+  if(log) console.log(newText)
+  newText = newText.replace(/<\/li><li>/g,'\n* ');
+  if(log) console.log(newText)
+  newText = newText.replace(/<\/li><\/ul>/g,'');
+  if(log) console.log(newText)
+  newText = newText.replace(/<br>/g,'\n\n');
+  if(log) console.log(newText)
+  return newText
+}
+processUnits = (data,assetCatalog) => {
+  var stringSimilarity = require('string-similarity');
+  data.datasheets.forEach(datasheet => {
+    let unitId = datasheet.datasheet_id;
+    let tempItem = {stats:{
+      'Power Level': {
+        value: Number(datasheet.power_points)
+      }
+    },keywords:{},assets:{}};
+
+    let models = data.models.filter(model => model.datasheet_id === unitId);
+    // console.log(unitId,models)
+    if(models[0]?.line === 1 && models[0]?.models_per_unit.includes('-')){
+      tempItem.stats.model = {
+        value: 'Model§' + data.models.filter(model => model.datasheet_id === unitId && model.line === 1)[0].name
+      }
+    }
+
+    let keywords = data.keywords.filter(keyword => keyword.datasheet_id === unitId && !keyword.is_faction_keyword);
+    if(keywords.length) tempItem.keywords.Keywords = keywords.map(keyword => keyword.keyword);
+    let factionKeywords = data.keywords.filter(keyword => keyword.datasheet_id === unitId && keyword.is_faction_keyword);
+    if(factionKeywords.length) tempItem.keywords.Faction = factionKeywords.map(keyword => keyword.keyword);
+
+    let options = data.options.filter(option => option.datasheet_id === unitId);
+    tempItem.text = formatText(datasheet.unit_composition + '\n\n' + options.map(option => (option.button || '') + ' ' + option.description).join('\n\n') + '\n\n' + datasheet.psyker);
+
+    if(models[0]?.models_per_unit?.includes('-')){
+      tempItem.stats[datasheet.name] = {
+        statType: 'numeric',
+        dynamic: true,
+        visibility: 'always',
+      };
+      let stat = tempItem.stats[datasheet.name];
+      let range = models[0].models_per_unit.split('-');
+      stat.value = Number(range[0]);
+      stat.min = Number(range[0]);
+      stat.max = Number(range[1]);
+      let basePlThresh = stat.min;
+      if(!(stat.max % stat.min)){
+        // console.log(datasheet.name,'has a clean threshold')
+        stat.increment = {value: stat.min};
+      }
+      else if(!((stat.max + 1) % (stat.min + 1)) && models[1]?.models_per_unit == 1){
+        // console.log(datasheet.name,'has a sergeant')
+        stat.increment = {value: stat.min + 1};
+        // console.log(stat)
+        basePlThresh ++;
+      }else tempItem.text += '\n\nERROR: there might be a problem with incrementation that will require inputting by hand.';
+      let PLArr = datasheet.unit_composition.split(/(\<b\>Power Rating |\<\/b\>)/).map(el => Number(el.replace('+','plus'))).filter(el => !isNaN(el));
+      if(PLArr.length){
+        let tempInc = PLArr[0] - datasheet.power_points;
+        // console.log(datasheet.name,basePlThresh,tempInc,PLArr)
+        for (let i = 0; i < PLArr.length; i++) {
+          if(PLArr[i] !== ((i+1) * tempInc) + Number(datasheet.power_points)){
+            // console.log(PLArr[i],tempInc,Number(datasheet.power_points), ((i+1) * tempInc) + Number(datasheet.power_points))
+            tempItem.text += '\n\nERROR: there might be a problem with Power Rating that will require a custom rule.';
+            tempInc = 0;
+            break;
+          }
+        }
+        if(tempInc){
+          tempItem.stats.poweri = {value:tempInc};
+          for (let i = 0; i < PLArr.length; i++) {
+            tempItem.stats['power'+(i+1)] = {
+              "value": (basePlThresh * (i + 1)) + 1
+            }
+          }
+        }
+      }else if(datasheet.unit_composition.includes('Power Rating')) tempItem.text += '\n\nERROR: there might be a problem with Power Rating that will require a custom rule.';
+    }
+    let modelList = [];
+    models.forEach(model => {
+      let [minQty,maxQty] = model.models_per_unit.split('-').map(qty => Number(qty));
+      let statlineName = model.itemKey.replace('Model§','Statline§');
+      if(minQty){
+        let defaultStatline = assetCatalog[statlineName];
+        // console.log(defaultStatline)
+        let tempTrait = {item: model.itemKey};
+        if(minQty > 1) tempTrait.quantity = minQty;
+        // console.log(datasheet.name,model.name,model.models_per_unit,models.length)
+        if(model.models_per_unit == 1 && models.length == 1){
+          tempTrait.stats = tempTrait.stats || {};
+          tempTrait.stats.Points = tempTrait.stats.Points || {};
+          tempTrait.stats.Points.visibility = 'hidden';
+        }
+        let tempStatline = {...tempTrait,item: statlineName};
+        delete tempStatline.quantity;
+        // console.log(tempTrait)
+        // console.log(tempStatline)
+        if(Object.keys(tempStatline).length === 1) tempStatline = statlineName;
+        if(Object.keys(tempTrait).length === 1) tempTrait = model.itemKey;
+        tempItem.assets.traits = tempItem.assets.traits || [];
+        tempItem.assets.traits.push(tempTrait);
+        modelList.push(model)
+        tempItem.assets.traits.push(tempStatline);
+      }
+      if(minQty > 1 || maxQty > 1){
+        tempItem.allowed = tempItem.allowed || {};
+        tempItem.allowed.items = tempItem.allowed.items || [];
+        tempItem.allowed.items.push(model.itemKey)
+      }
+    });
+
+    let abilities = data.abilities.composed.filter(ability => ability.datasheet_id === unitId);
+    let abilityList = abilities.filter(ability => ability.datasheet_id === unitId && !ability.is_other_wargear);
+    let wargearList = abilities.filter(ability => ability.datasheet_id === unitId && ability.is_other_wargear);
+    abilityList.forEach(ability => {
+      tempItem.assets = tempItem.assets || {};
+      tempItem.assets.traits = tempItem.assets.traits || [];
+      tempItem.assets.traits.push(ability.itemKey);
+    });
+    const order = ['Statline§', 'Ability§', 'Wargear§', 'Psychic Power§', 'Model§'];
+    tempItem.assets.traits.sort((a, b) => stringSimilarity.findBestMatch((a.item || a),order).bestMatchIndex - stringSimilarity.findBestMatch((b.item || b),order).bestMatchIndex);
+
+    if(datasheet.psyker?.includes('Smite')){
+      tempItem.assets = tempItem.assets || {};
+      tempItem.assets.traits = tempItem.assets.traits || [];
+      tempItem.assets.traits.push('Psychic Power§Smite');
+    }
+    Array.from(new Set(data.psychicPowers.map(power => power.type))).forEach(discipline => {
+      // console.log(discipline)
+      // console.log(datasheet.psyker)
+      let test = new RegExp(discipline,'gi')
+      if(test.test(datasheet.psyker)){
+        tempItem.allowed = tempItem.allowed || {};
+        tempItem.allowed.classifications = tempItem.allowed.classifications || [];
+        tempItem.allowed.classifications.push(discipline);
+      }
+    });
+
+    wargearList.forEach(wargear => {
+      let tempWargear = wargear.cost === assetCatalog[wargear.itemKey].stats?.Points?.value ? wargear.itemKey : {
+        item: wargear.itemKey,
+        stats: {
+          Points: {value: wargear.cost}
+        }
+      };
+      tempItem.stats = tempItem.stats || {};
+      tempItem.stats[wargear.name] = {
+        value: 0,
+        statType: 'rank',
+        statOrder: 10,
+        ranks: {
+          0: {order: 0,number: 0,icons: ['cancel']},
+          1: {order: 1,number: 1,icons: ['confirmed'],traits: [{trait: tempWargear}]}
+        },
+        visibility: 'active',
+        dynamic: true
+      }
+    });
+    // console.log(datasheet.name,unitId,wargearList)
+
+    let wargearArr = data.wargear.composed.filter(wargear => wargear.datasheet_id == unitId).sort((a,b) => a.itemKey.localeCompare(b.itemKey));
+    wargearArr.slice().forEach((gear,i) => {
+      if(wargearArr[i].itemKey?.includes(' (melee)') && wargearArr[i+1]?.includes(' (shooting)')){
+        wargearArr[i].itemKey = wargearArr[i].itemKey.replace('Weapon§','Wargear§').replace(' (melee)','');
+        delete wargearArr[i+1];
+      }
+    });
+    wargearArr = Object.values(wargearArr);
+    let equippedWargearArr = datasheet.unit_composition?.replace(/is equipped<br>with/g,'is equipped with').replace(/(<br>|<ul><li>|<li><li>|<\/li><\/ul>|<\/b> |\.\s)/g,'. ').split('. ').filter(el => el.includes('is equipped')).map(el => el.split(/is equipped with/).map(subEl => subEl.split('; ').map(equip => equip.replace(/^([:Aa1]\s)*/,''))));
+    // console.log(datasheet.name,unitId,wargearArr,datasheet.unit_composition)
+    equippedWargearArr?.forEach(modelLoadout => {
+      // console.log(modelLoadout[0][0],modelLoadout[1])
+      if(!modelLoadout[1]?.includes(' nothing.')){
+        let upgradeQty = modelLoadout[1]?.length ? (modelLoadout[1].length + 1) : 0;
+        if(
+          stringSimilarity.compareTwoStrings(modelLoadout[0][0],'Every model') > .5
+          || stringSimilarity.compareTwoStrings(modelLoadout[0][0],'Each model') > .5
+          || stringSimilarity.compareTwoStrings(modelLoadout[0][0],'This model') > .5
+        ){
+          models.forEach(modelData => {
+            let tempItem = assetCatalog[modelData.itemKey];
+            if(upgradeQty) tempItem.stats = tempItem.stats || {};
+            for (let i = 0; i < upgradeQty; i++) {
+              tempItem.stats['loadout'+(i+1)] = createWargearStat(i,wargearArr,modelLoadout[1],assetCatalog);
+              // console.log(tempItem,upgradeQty)
+            }
+          });
+        }else{
+          let modelNames = models.map(thisModel => thisModel.name);
+          let modelIndex = stringSimilarity.findBestMatch(modelLoadout[0][0],modelNames).bestMatchIndex;
+          let tempItem = assetCatalog[models[modelIndex].itemKey];
+          if(upgradeQty) tempItem.stats = tempItem.stats || {};
+          for (let i = 0; i < upgradeQty; i++) {
+            tempItem.stats['loadout'+(i+1)] = createWargearStat(i,wargearArr,modelLoadout[1],assetCatalog);
+            // console.log(tempItem,upgradeQty)
+          }
+        }
+      }
+    });
+
+    let source = data.sources.filter(source => source.source_id == datasheet.source_id)[0];
+    // console.log(source)
+    if(source){
+      let errataDate = source.errata_date.split(' ')[0].split('.').reverse().join('-');
+      tempItem.meta = tempItem.meta || {};
+      tempItem.meta.Publication = `[${source.name} (${source.type}) ${ordinalize(source.edition)} ed. – ${source.version || ''} @${errataDate}](${source.errata_link})`;
+    }
+    // TODO implement dynamic stats
+
+    let modelDamage = data.damage.filter(dmgLine => dmgLine.datasheet_id == unitId);
+    if(modelDamage.length){
+      let modelItemKey = models.filter(model => model.datasheet_id === unitId)[0].itemKey;
+      // console.log(unitId,modelItemKey)
+      assetCatalog[modelItemKey].stats['W'] = {
+        value: assetCatalog[modelItemKey].stats['W'].value,
+        max: assetCatalog[modelItemKey].stats['W'].value,
+        min: 1,
+        dynamic: true,
+        increment: {value: 1},
+        statType: 'numeric',
+        visibility: 'always',
+      }
+      assetCatalog[modelItemKey].rules = assetCatalog[modelItemKey].rules || {};
+      assetCatalog[modelItemKey].rules.dynamicDamageMid = generateDamageRule(modelDamage[0],modelDamage[2]);
+      assetCatalog[modelItemKey].rules.dynamicDamageLow = generateDamageRule(modelDamage[0],modelDamage[3]);
+    }
+
+    assetCatalog[datasheet.role + '§' + datasheet.name] = tempItem;
+  });
+}
+generateDamageRule = (damageRows,currentRow) => {
+  // console.log(damageRows,currentRow)
+  let [min,max] = currentRow.col1.split('-');
+  let newRule = {
+    evals: [
+      {
+        paths: [
+          ['{self}','stats','W','value']
+        ],
+        max: max,
+        min: min,
+        operator: 'AND',
+        not: false,
+        actionable: true
+      }
+    ],
+    failState: 'pass',
+    evaluate: 'AND',
+    actions: [
+      {
+        paths: [
+          ['{self}','stats',damageRows.col2,'value']
+        ],
+        actionType: 'set',
+        value: typeof currentRow.col2 === 'number' ? currentRow.col2 : currentRow.col2?.replace('"',''),
+        iterations: 1
+      },
+      {
+        paths: [
+          ['{self}','stats',damageRows.col3,'value']
+        ],
+        actionType: 'set',
+        value: typeof currentRow.col3 === 'number' ? currentRow.col3 : currentRow.col3?.replace('"',''),
+        iterations: 1
+      },
+      {
+        paths: [
+          ['{self}','stats',damageRows.col4,'value']
+        ],
+        actionType: 'set',
+        value: typeof currentRow.col4 === 'number' ? currentRow.col4 : currentRow.col4?.replace('"',''),
+        iterations: 1
+      }
+    ]
+  }
+  return newRule
+}
+processFactions = (data,assetTaxonomy) => {
+  let fac = data.factions[0].main_faction_id;
+  // console.log(fac,data.factions.length)
+  data.factCurrent = subFacNames[fac];
+  if(data.factions.length > 1){
+    assetTaxonomy.Detachment = {
+      stats: {
+        [subFacNames[fac]]: {
+          statType: 'rank',
+          value: '-',
+          ranks: {
+            '-': {
+              order: 0
+            },
+          },
+          dynamic: true,
+        }
+      },
+      rules: {
+        'populate faction': {
+          evals: [
+            {
+              paths: [
+                ["{self}","stats","Brotherhood","value"]
+              ],
+              value: "-",
+              operator: "AND",
+              not: true,
+              actionable: true
+            }
+          ],
+          failState: 'pass',
+          evaluate: 'OR',
+          actions: [
+            {
+              paths: [
+                [
+                  '{self}',
+                  'assets',
+                  'templateClass',
+                  'Unit',
+                  'traits',
+                  'classification',
+                  'Ability',
+                  'stats',
+                  subFacNames[fac],
+                  'value',
+                ]
+              ],
+              actionType: 'set',
+              value: [
+                '{self}',
+                'stats',
+                subFacNames[fac],
+                'value',
+              ],
+              iterations: 1
+            }
+          ]
+        }
+      }
+    }
+    data.factions.filter(faction => faction.faction_id != faction.main_faction_id).forEach((faction,i) => {
+      let newRank = {order:i+1}
+      assetTaxonomy.Detachment.stats[subFacNames[fac]].ranks[faction.name] = newRank;
+    });
+    assetTaxonomy.Unit = {
+      rules: {
+        'replace subfaction keyword': {
+          evals: [
+            {
+              paths: [
+                ['{parent}','stats',data.factCurrent,'value']
+              ],
+              value: '-',
+              operator: 'AND',
+              not: true
+            },
+            {
+              paths: [
+                ['{self}','keywords','Faction']
+              ],
+              value: `<${data.factCurrent}>`,
+              contains: true,
+              operator: 'AND',
+              not: false,
+              actionable: true
+            }
+          ],
+          failState: 'pass',
+          evaluate: 'AND',
+          actions: [
+            {
+              paths: [
+                ['{self}','keywords','Faction']
+              ],
+              actionType: 'remove',
+              value: `<${data.factCurrent}>`,
+              iterations: 1
+            },
+            {
+              paths: [
+                ['{self}','keywords','Faction']
+              ],
+              actionType: 'add',
+              value: ['{parent}','stats',data.factCurrent,'processed','rank','current'],
+              iterations: 1
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+processPsychicClasses = (data,assetTaxonomy) => {
+  data.psychicPowers.forEach(power => {
+    if(power.type){
+      assetTaxonomy[power.type] = assetTaxonomy[power.type] || {
+        templateClass: 'Psychic Power',
+        stats:{Roll:{
+          value: null
+        }}
+      }
+    }
+  });
+}
+module.exports = { processItems };

wahapedia_import/exportion/public/stylesheets/style.css

@@ -0,0 +1,8 @@
+body {
+  padding: 50px;
+  font: 14px "Lucida Grande", Helvetica, Arial, sans-serif;
+}
+
+a {
+  color: #00B7FF;
+}

wahapedia_import/exportion/routes/index.js

@@ -0,0 +1,270 @@
+var express = require('express');
+var router = express.Router();
+const pool = require('../config/data');
+const SqlString = require('sqlstring');
+const { processItems } = require('../public/javascripts/manifest.process');
+
+/* GET home page. */
+router.get('/', function(req, res, next) {
+  console.log(req.body)
+  // var sql = SqlString.format("SELECT * FROM datasheets;");
+  var sql = SqlString.format("SELECT * FROM factions;");
+  console.log(sql)
+  pool.query(sql, function (error, results, fields) {
+    if(error) {
+      console.log(error);
+      res.send(sql);
+      return;
+    }
+    console.log(results)
+    let factionList = results.filter(faction => faction.faction_id === faction.main_faction_id).sort((a,b) => a.name.localeCompare(b.name));
+    console.log(factionList)
+    
+    //if we actually get a result
+    
+    res.render('index', { title: 'Wahapedia Exportion', data: factionList});
+    // res.send(JSON.stringify(results.map(datasheet => `${datasheet.datasheet_id}: ${datasheet.name} – ${datasheet.unit_composition}`))); 
+  });
+});
+router.get('/favicon.ico', function(req, res, next) {})
+
+router.get('/:faction', async function (req, res, next) {
+  console.log('req.body',req.body)
+  let sql = '';
+  let allResults = {};
+  allResults.factions = await getFactions(req.params.faction);
+  allResults.datasheets = await getDatasheets(req.params.faction);
+  let datasheetList = Array.from(new Set(allResults.datasheets.map(datasheet => datasheet.datasheet_id)));
+  console.log(datasheetList)
+  allResults.keywords = await getKeywords(datasheetList);
+  allResults.models = await getModels(datasheetList);
+  allResults.damage = await getDamage(datasheetList);
+  allResults.wargear = await getWargear(datasheetList);
+  allResults.abilities = await getAbilities(datasheetList);
+  allResults.options = await getOptions(datasheetList);
+  allResults.psychicPowers = await getPsychicPowers(req.params.faction);
+  allResults.stratagems = await getStrategems(req.params.faction);
+  allResults.warlordTraits = await getWarlordTraits(req.params.faction);
+  allResults.sources = await getSources();
+  // console.log('allresults',allResults)
+  allResults['!'] = processInfo(allResults,req.params.faction);
+  allResults['!'].manifest.assetTaxonomy = processClasses(allResults);
+  allResults['!'].manifest.assetCatalog = processItems(allResults);
+  res.send(JSON.stringify(allResults['!']));
+});
+
+
+let getFactions = async (fac) => {
+  const sql = SqlString.format("SELECT * FROM factions WHERE main_faction_id = ?",[fac]);
+  console.log('query',fac,sql)
+  let results = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  // console.log('factions',results)
+  return results;
+}
+
+let getDatasheets = async (fac) => {
+  const sql = SqlString.format("SELECT * FROM datasheets WHERE faction_id = ?",[fac]);
+  let results = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  // console.log('datasheets',results)
+  return results;
+}
+
+let getKeywords = async (datasheets) => {
+  const sql = SqlString.format("SELECT * FROM datasheets_keywords WHERE datasheet_id in (?)",[datasheets]);
+  console.log('query',sql)
+  let results = datasheets.length ? await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  })) : {};
+  // console.log('keywords',results)
+  return results;
+}
+
+let getModels = async (datasheets) => {
+  const sql = SqlString.format("SELECT * FROM datasheets_models WHERE datasheet_id in (?)",[datasheets]);
+  console.log('query',sql)
+  let results = datasheets.length ? await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  })) : {};
+  // console.log('models',results)
+  return results;
+}
+
+let getDamage = async (datasheets) => {
+  const sql = SqlString.format("SELECT * FROM datasheets_damage WHERE datasheet_id in (?)",[datasheets]);
+  console.log('query',sql)
+  let results = datasheets.length ? await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  })) : {};
+  // console.log('models',results)
+  return results;
+}
+
+let getWargear = async (datasheets) => {
+  let sql = SqlString.format("SELECT * FROM datasheets_wargear WHERE datasheet_id in (?)",[datasheets]);
+  let datasheets_wargear = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  let wargearIDs = Array.from(new Set(datasheets_wargear?.map(wargear => wargear?.wargear_id)));
+  sql = SqlString.format("SELECT * FROM wargear_list WHERE wargear_id in (?)",[wargearIDs]);
+  let wargear_list = wargearIDs.length ? await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  })) : {};
+  sql = SqlString.format("SELECT * FROM wargear WHERE wargear_id in (?)",[wargearIDs]);
+  let wargear = wargearIDs.length ? await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  })) : {};
+  // console.log('models',results)
+  return {
+    datasheets_wargear:datasheets_wargear,
+    wargear_list:wargear_list,
+    wargear:wargear,
+  };
+}
+
+let getAbilities = async (datasheets) => {
+  let sql = SqlString.format("SELECT * FROM datasheets_abilities WHERE datasheet_id in (?)",[datasheets]);
+  let datasheets_abilities = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  let abilityIDs = Array.from(new Set(datasheets_abilities.map(ability => ability.ability_id)));
+  sql = SqlString.format("SELECT * FROM abilities WHERE ability_id in (?)",[abilityIDs]);
+  let abilities = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  // console.log('models',results)
+  return {
+    datasheets_abilities:datasheets_abilities,
+    abilities:abilities,
+  };
+}
+let getOptions = async (datasheets) => {
+  const sql = SqlString.format("SELECT * FROM datasheets_options WHERE datasheet_id in (?)",[datasheets]);
+  let results = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  return results;
+}
+let getPsychicPowers = async (fac) => {
+  const sql = SqlString.format("SELECT * FROM psychic_powers WHERE faction_id = ?",[fac]);
+  let results = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  return results;
+}
+let getStrategems = async (fac) => {
+  const sql = SqlString.format("SELECT * FROM strategems WHERE faction_id = ?",[fac]);
+  let results = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  return results;
+}
+let getWarlordTraits = async (fac) => {
+  const sql = SqlString.format("SELECT * FROM warlord_traits WHERE faction_id = ?",[fac]);
+  let results = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  return results;
+}
+let getSources = async () => {
+  const sql = SqlString.format("SELECT * FROM sources");
+  let results = await new Promise((resolve, reject) => pool.query(sql, (error, results) => {
+    if(error) {
+      console.log(error);
+      reject(error);
+      return;
+    }else{
+      resolve(results);
+    }
+  }));
+  return results;
+}
+
+module.exports = router;

wahapedia_import/exportion/views/error.jade

@@ -0,0 +1,6 @@
+extends layout
+
+block content
+  h1= message
+  h2= error.status
+  pre #{error.stack}

wahapedia_import/exportion/views/index.jade

@@ -0,0 +1,7 @@
+extends layout
+
+block content
+  h1= title
+  each faction in data
+    p
+      a(href=`/${faction.faction_id}`)= faction.name

wahapedia_import/exportion/views/layout.jade

@@ -0,0 +1,7 @@
+doctype html
+html
+  head
+    title= title
+    link(rel='stylesheet', href='/stylesheets/style.css')
+  body
+    block content

wahapedia_import/ingestion.py

@@ -0,0 +1,959 @@
+import mysql.connector
+from mysql.connector import errorcode
+import re
+import config
+import csv
+
+try:
+  db = mysql.connector.connect(
+    host=config.host,
+    user=config.user,
+    passwd=config.passwd,
+    database=config.database
+  )
+except mysql.connector.Error as err:
+  if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
+    print("Something is wrong with your user name or password")
+  elif err.errno == errorcode.ER_BAD_DB_ERROR:
+    print("Database does not exist")
+  else:
+    print(err)
+else:
+  cur = db.cursor(named_tuple=True)
+  
+  try:
+    with open('wahapediaSetupDatabase.sql',mode='r') as f:
+      resItr = cur.execute(f.read(), multi=True)
+      for res in resItr:
+        print("Running query: ", res)
+        print(f"Affected {res.rowcount} rows" )
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to load wahapediaSetupDatabase.sql")
+    
+  
+  # ORDER OF DATA INSERTION
+  # sources;
+  # factions;
+  # wargear;
+  # wargear_list;
+  # datasheets;
+  # abilities;
+  # datasheets_abilities;
+  # psychic_powers;
+  # warlord_traits;
+  # strategems;
+  # datasheets_damage;
+  # datasheets_keywords;
+  # datasheets_models;
+  # datasheets_options;
+  # datasheets_wargear;
+  
+  # all the data has an unused column at the end, which is why every processor cuts off the last column
+  # MySQL doesn't have the boolean datatype, we're using INT with 1 = true, 0 = false, -1 = data input error
+  print("\nSOURCES")
+  try:
+    with open("wahapedia/Source.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "id"):
+          indexer["source_id"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "type"):
+          indexer["type"] = i
+        elif(column == "edition"):
+          indexer["edition"] = i
+        elif(column == "version"):
+          indexer["version"] = i
+        elif(column == "errata_date"):
+          indexer["errata_date"] = i
+        elif(column == "errata_link"):
+          indexer["errata_link"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO sources (source_id, name, type, edition, version, errata_date, errata_link) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s),"
+        data = data + [
+          row[indexer["source_id"]] if row[indexer["source_id"]] != '' else None, # source_id
+          row[indexer["name"]] if row[indexer["name"]] != '' else None, # name
+          row[indexer["type"]] if row[indexer["type"]] != '' else None, # type
+          row[indexer["edition"]] if row[indexer["edition"]] != '' else None, # edition
+          row[indexer["version"]] if row[indexer["version"]] != '' else None, # version
+          row[indexer["errata_date"]] if row[indexer["errata_date"]] != '' else None, # errata_date
+          row[indexer["errata_link"]] if row[indexer["errata_link"]] != '' else None, # errata_link
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into sources")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Source.csv")
+    
+  print("\nFACTIONS")
+  try:
+    with open("wahapedia/Factions.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "id"):
+          indexer["faction_id"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "link"):
+          indexer["link"] = i
+        elif(column == "is_subfaction"):
+          indexer["is_subfaction"] = i
+        i = i + 1
+      
+      primary_factions = {}
+      
+      query = "INSERT IGNORE INTO factions (faction_id, name, link, is_subfaction, main_faction_id) VALUES "
+      data = []
+      for row in csvF:
+        
+        main_faction_id = None
+        if(re.sub('/.*$', '', re.sub('https://wahapedia.ru/wh40k9ed/factions/','',row[indexer["link"]])) == re.sub('[^0-9a-zA-Z]+', '-', row[indexer["name"]].lower())):
+          main_faction_id = row[indexer["faction_id"]]
+          primary_factions[row[indexer["link"]]] = main_faction_id
+          
+        query+="(%s,%s,%s,%s,%s),"
+        # data has duplicate information
+        data = data + [
+          row[indexer["faction_id"]] if row[indexer["faction_id"]] != '' else None, # faction_id
+          row[indexer["name"]] if row[indexer["name"]] != '' else None, # name
+          row[indexer["link"]] if row[indexer["link"]] != '' else None, # link
+          row[indexer["is_subfaction"]] if row[indexer["is_subfaction"]] != '' else None, # link
+          main_faction_id, # effectively foreign key to this table's faction_id
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into factions")
+      db.commit()
+      
+      # updating non primary_factions with their associated primary_faction's faction_id
+      cur.execute("UPDATE factions f1 INNER JOIN factions f2 ON f1.link LIKE CONCAT(f2.link, '%') SET f1.main_faction_id = f2.main_faction_id WHERE f2.main_faction_id IS NOT NULL;")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Factions.csv")
+  
+  print("\nWARGEAR")
+  try:
+    with open("wahapedia/Wargear.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "id"):
+          indexer["wargear_id"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "type"):
+          indexer["type"] = i
+        elif(column == "description"):
+          indexer["description"] = i
+        elif(column == "is_relic"):
+          indexer["is_relic"] = i
+        elif(column == "faction_id"):
+          indexer["faction_id"] = i
+        elif(column == "legend"):
+          indexer["legend"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO wargear (wargear_id, name, type, description, is_relic, faction_id, legend) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s),"
+        # data has unused column
+        data = data + [
+          row[indexer["wargear_id"]] if row[indexer["wargear_id"]] != '' else None, # wargear_id
+          row[indexer["name"]] if row[indexer["name"]] != '' else None, # name
+          row[indexer["type"]] if row[indexer["type"]] != '' else None, # type
+          row[indexer["description"]] if row[indexer["description"]] != '' else None, # description
+          1 if row[indexer["is_relic"]] == "true" else 0 if row[indexer["is_relic"]] == "false" else -1, # is_relic, boolean
+          row[indexer["faction_id"]] if row[indexer["faction_id"]] != '' else None, # faction_id
+          row[indexer["legend"]] if row[indexer["legend"]] != '' else None, # legend
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into wargear")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Wargear.csv")
+  
+  print("\nWARGEAR LIST")
+  try:
+    with open("wahapedia/Wargear_list.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      query = "INSERT IGNORE INTO wargear_list (wargear_id, line, name, weapon_range, type, strength, armor_piercing, damage, abilities) VALUES "
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "wargear_id"):
+          indexer["wargear_id"] = i
+        elif(column == "line"):
+          indexer["line"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "Range"):
+          indexer["weapon_range"] = i
+        elif(column == "type"):
+          indexer["type"] = i
+        elif(column == "S"):
+          indexer["strength"] = i
+        elif(column == "AP"):
+          indexer["armor_piercing"] = i
+        elif(column == "D"):
+          indexer["damage"] = i
+        elif(column == "abilities"):
+          indexer["abilities"] = i
+        i = i + 1
+      
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s,%s,%s),"
+        data = data + [
+          row[indexer["wargear_id"]] if row[indexer["wargear_id"]] != '' else None, # wargear_id
+          row[indexer["line"]] if row[indexer["line"]] != '' else None, # line
+          row[indexer["name"]] if row[indexer["name"]] != '' else None, # name
+          row[indexer["weapon_range"]].replace('"', '″') if row[indexer["weapon_range"]] != '' else None, # weapon_range
+          row[indexer["type"]] if row[indexer["type"]] != '' else None, # type
+          row[indexer["strength"]] if row[indexer["strength"]] != '' else None, # strength
+          row[indexer["armor_piercing"]] if row[indexer["armor_piercing"]] != '' else None, # armor_piercing
+          row[indexer["damage"]] if row[indexer["damage"]] != '' else None, # damage
+          row[indexer["abilities"]] if row[indexer["abilities"]] != '' else None, # abilities
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into wargear_list")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Wargear_list.csv")
+  
+  print("\nDATASHEETS")
+  try:
+    with open("wahapedia/Datasheets.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "id"):
+          indexer["datasheet_id"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "link"):
+          indexer["link"] = i
+        elif(column == "faction_id"):
+          indexer["faction_id"] = i
+        elif(column == "source_id"):
+          indexer["source_id"] = i
+        elif(column == "role"):
+          indexer["role"] = i
+        elif(column == "unit_composition"):
+          indexer["unit_composition"] = i
+        elif(column == "transport"):
+          indexer["transport"] = i
+        elif(column == "power_points"):
+          indexer["power_points"] = i
+        elif(column == "priest"):
+          indexer["priest"] = i
+        elif(column == "psyker"):
+          indexer["psyker"] = i
+        elif(column == "open_play_only"):
+          indexer["open_play_only"] = i
+        elif(column == "crusade_only"):
+          indexer["crusade_only"] = i
+        elif(column == "virtual"):
+          indexer["virtual_"] = i
+        elif(column == "Cost"):
+          indexer["cost"] = i
+        elif(column == "cost_per_unit"):
+          indexer["cost_per_unit"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO datasheets (datasheet_id, name, link, faction_id, source_id, role, unit_composition, transport, power_points, priest, psyker, open_play_only, crusade_only, virtual_, cost, cost_per_unit) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s),"
+        data = data + [
+          row[indexer["datasheet_id"]] if row[indexer["datasheet_id"]] != '' else None,  # datasheet_id
+          row[indexer["name"]] if row[indexer["name"]] != '' else None,  # name
+          row[indexer["link"]] if row[indexer["link"]] != '' else None,  # link
+          row[indexer["faction_id"]] if row[indexer["faction_id"]] != '' else None,  # faction_id
+          row[indexer["source_id"]] if row[indexer["source_id"]] != '' else None,  # source_id
+          row[indexer["role"]] if row[indexer["role"]] != '' else None,  # role
+          row[indexer["unit_composition"]] if row[indexer["unit_composition"]] != '' else None,  # unit_composition
+          row[indexer["transport"]] if row[indexer["transport"]] != '' else None,  # transport
+          row[indexer["power_points"]] if row[indexer["power_points"]] != '' else None,  # power_points
+          row[indexer["priest"]] if row[indexer["priest"]] != '' else None,  # priest
+          row[indexer["psyker"]] if row[indexer["psyker"]] != '' else None, # psyker
+          1 if row[indexer["open_play_only"]] == "true" else 0 if row[indexer["open_play_only"]] == "false" else -1, # open_play_only, boolean
+          1 if row[indexer["crusade_only"]] == "true" else 0 if row[indexer["crusade_only"]] == "false" else -1, # crusade_only, boolean
+          1 if row[indexer["virtual_"]] == "true" else 0 if row[indexer["virtual_"]] == "false" else -1, # virtual_, boolean
+          row[indexer["cost"]] if row[indexer["cost"]] != '' else None, # cost
+          1 if row[indexer["cost_per_unit"]] == "true" else 0 if row[indexer["cost_per_unit"]] == "false" else -1, # cost_per_unit, boolean
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into datasheets")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Datasheets.csv")
+  
+  print("\nABILITIES")
+  try:
+    with open("wahapedia/Abilities.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "id"):
+          indexer["ability_id"] = i
+        elif(column == "type"):
+          indexer["type"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "legend"):
+          indexer["legend"] = i
+        elif(column == "is_other_wargear"):
+          indexer["is_other_wargear"] = i
+        elif(column == "faction_id"):
+          indexer["faction_id"] = i
+        elif(column == "description"):
+          indexer["description"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO abilities (ability_id, type, name, legend, is_other_wargear, faction_id, description) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s),"
+        data = data + [
+          row[indexer["ability_id"]], # ability_id
+          row[indexer["type"]], # type
+          row[indexer["name"]], # name
+          row[indexer["legend"]], # legend
+          1 if row[indexer["is_other_wargear"]] == "true" else 0 if row[indexer["is_other_wargear"]] == "false" else -1, # is_other_wargear, boolean
+          row[indexer["faction_id"]], # faction_id,
+          row[indexer["description"]], # description
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into abilities")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Abilities.csv")
+  
+  print("\nDATASHEETS ABILITIES")
+  try:
+    with open("wahapedia/Datasheets_abilities.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "datasheet_id"):
+          indexer["datasheet_id"] = i
+        elif(column == "line"):
+          indexer["line"] = i
+        elif(column == "ability_id"):
+          indexer["ability_id"] = i
+        elif(column == "is_index_wargear"):
+          indexer["is_index_wargear"] = i
+        elif(column == "cost"):
+          indexer["cost"] = i
+        elif(column == "model"):
+          indexer["model"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO datasheets_abilities (datasheet_id, line, ability_id, is_index_wargear, cost, model) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s),"
+        data = data + [
+          row[indexer["datasheet_id"]], # datasheet_id
+          row[indexer["line"]], # line
+          row[indexer["ability_id"]], # ability_id
+          1 if row[indexer["is_index_wargear"]] == "true" else 0 if row[indexer["is_index_wargear"]] == "false" else -1, # is_index_wargear, boolean
+          row[indexer["cost"]], # cost
+          row[indexer["model"]], # model
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into datasheets_abilities")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Datasheets_abilities.csv")
+  
+  print("\nPSYCHIC POWERS")
+  try:
+    with open("wahapedia/PsychicPowers.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "id"):
+          indexer["psychic_power_id"] = i
+        elif(column == "roll"):
+          indexer["roll"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "faction_id"):
+          indexer["faction_id"] = i
+        elif(column == "legend"):
+          indexer["legend"] = i
+        elif(column == "type"):
+          indexer["type"] = i
+        elif(column == "description"):
+          indexer["description"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO psychic_powers (psychic_power_id, roll, name, faction_id, legend, type, description) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s),"
+        # input data is out of order (doesn't start with id) and contains duplicate information
+        data = data + [
+          row[indexer["psychic_power_id"]], # psychic_power_id
+          row[indexer["roll"]], # roll
+          row[indexer["name"]], # name
+          row[indexer["faction_id"]], # faction_id
+          row[indexer["legend"]], # legend
+          row[indexer["type"]], # type
+          row[indexer["description"]], # description
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into psychic_powers")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/PsychicPowers.csv")
+  
+  print("\nWARLORD TRAITS")
+  try:
+    with open("wahapedia/Warlord_traits.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "faction_id"):
+          indexer["faction_id"] = i
+        elif(column == "type"):
+          indexer["type"] = i
+        elif(column == "roll"):
+          indexer["roll"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "legend"):
+          indexer["legend"] = i
+        elif(column == "description"):
+          indexer["description"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO warlord_traits (trait_id, faction_id, type, roll, name, legend, description) VALUES "
+      data = []
+      for row in csvF:
+        # data doesn't have ID for rows
+        query+="(DEFAULT,%s,%s,%s,%s,%s,%s),"
+        # data has duplicate information
+        data = data + [
+          row[indexer["faction_id"]], # faction_id
+          row[indexer["type"]], # type
+          row[indexer["roll"]], # roll
+          row[indexer["name"]], # name
+          row[indexer["legend"]], # legend
+          row[indexer["description"]], # description
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into warlord_traits")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Warlord_traits.csv")
+  
+  print("\nSTRATAGEMS")
+  try:
+    with open("wahapedia/Stratagems.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "id"):
+          indexer["strategem_id"] = i
+        elif(column == "faction_id"):
+          indexer["faction_id"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "type"):
+          indexer["type"] = i
+        elif(column == "cp_cost"):
+          indexer["cp_cost"] = i
+        elif(column == "legend"):
+          indexer["legend"] = i
+        elif(column == "source_id"):
+          indexer["source_id"] = i
+        elif(column == "description"):
+          indexer["description"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO strategems (strategem_id, faction_id, name, type, cp_cost, legend, source_id, description) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s,%s),"
+        # input data is out of order (doesn't start with id)
+        data = data + [
+          row[indexer["strategem_id"]], # strategem_id
+          row[indexer["faction_id"]], # faction_id
+          row[indexer["name"]], # name
+          row[indexer["type"]], # type
+          row[indexer["cp_cost"]], # cp_cost
+          row[indexer["legend"]], # legend
+          row[indexer["source_id"]], # source_id
+          row[indexer["description"]], # description
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into strategems")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Stratagems.csv")
+  
+  print("\nDATASHEETS DAMAGE")
+  try:
+    with open("wahapedia/Datasheets_damage.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "datasheet_id"):
+          indexer["datasheet_id"] = i
+        elif(column == "line"):
+          indexer["line"] = i
+        elif(column == "Col1"):
+          indexer["col1"] = i
+        elif(column == "Col2"):
+          indexer["col2"] = i
+        elif(column == "Col3"):
+          indexer["col3"] = i
+        elif(column == "Col4"):
+          indexer["col4"] = i
+        elif(column == "Col5"):
+          indexer["col5"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO datasheets_damage (datasheet_id, line, col1, col2, col3, col4, col5) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s),"
+        # every col column is highly variable
+        data = data + [
+          row[indexer["datasheet_id"]], # datasheet_id
+          row[indexer["line"]], # line
+          row[indexer["col1"]].replace('+', ''), # col1
+          row[indexer["col2"]].replace('+', ''), # col2
+          row[indexer["col3"]].replace('+', ''), # col3
+          row[indexer["col4"]].replace('+', ''), # col4
+          row[indexer["col5"]].replace('+', ''), # col5
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into datasheets_damage")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Datasheets_damage.csv")
+  
+  print("\nDATASHEETS KEYWORDS")
+  try:
+    with open("wahapedia/Datasheets_keywords.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "datasheet_id"):
+          indexer["datasheet_id"] = i
+        elif(column == "keyword"):
+          indexer["keyword"] = i
+        elif(column == "model"):
+          indexer["model"] = i
+        elif(column == "is_faction_keyword"):
+          indexer["is_faction_keyword"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO datasheets_keywords (datasheet_id, keyword, model, is_faction_keyword) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s),"
+        data = data + [
+          row[indexer["datasheet_id"]], # datasheet_id
+          row[indexer["keyword"]], # keyword
+          row[indexer["model"]] if row[indexer["model"]] != '' else None, # model
+          1 if row[indexer["is_faction_keyword"]] == "true" else 0 if row[indexer["is_faction_keyword"]] == "false" else -1, # is_faction_keyword, boolean
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into datasheets_keywords")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Datasheets_keywords.csv")
+  
+  print("\nDATASHEETS MODELS")
+  try:
+    with open("wahapedia/Datasheets_models.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "datasheet_id"):
+          indexer["datasheet_id"] = i
+        elif(column == "line"):
+          indexer["line"] = i
+        elif(column == "name"):
+          indexer["name"] = i
+        elif(column == "M"):
+          indexer["movement"] = i
+        elif(column == "WS"):
+          indexer["weapon_skill"] = i
+        elif(column == "BS"):
+          indexer["ballistic_skill"] = i
+        elif(column == "S"):
+          indexer["strength"] = i
+        elif(column == "T"):
+          indexer["toughness"] = i
+        elif(column == "W"):
+          indexer["wounds"] = i
+        elif(column == "A"):
+          indexer["attacks"] = i
+        elif(column == "Ld"):
+          indexer["leadership"] = i
+        elif(column == "Sv"):
+          indexer["save"] = i
+        elif(column == "Cost"):
+          indexer["cost"] = i
+        elif(column == "cost_description"):
+          indexer["cost_description"] = i
+        elif(column == "models_per_unit"):
+          indexer["models_per_unit"] = i
+        elif(column == "cost_including_wargear"):
+          indexer["cost_including_wargear"] = i
+        elif(column == "base_size"):
+          indexer["base_size"] = i
+        elif(column == "base_size_descr"):
+          indexer["base_size_descr"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO datasheets_models (datasheet_id, line, name, movement, weapon_skill, ballistic_skill, strength, toughness, wounds, attacks, leadership, save, cost, cost_description, models_per_unit, cost_including_wargear, base_size, base_size_descr) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s),"
+        data = data + [
+          row[indexer["datasheet_id"]], # datasheet_id
+          row[indexer["line"]], # line
+          row[indexer["name"]], # name
+          row[indexer["movement"]].replace('"', ''), # movement
+          row[indexer["weapon_skill"]][0] if row[indexer["weapon_skill"]][0] != '-' else None, # weapon_skill, deformatted
+          row[indexer["ballistic_skill"]][0] if row[indexer["ballistic_skill"]][0] != '-' else None, # ballistic_skill, deformatted
+          row[indexer["strength"]] if row[indexer["strength"]][0] != '-' else None, # strength, formatted
+          row[indexer["toughness"]] if row[indexer["toughness"]][0] != '-' else None, # toughness, formatted
+          row[indexer["wounds"]] if row[indexer["wounds"]][0] != '-' else None, # wounds, formatted
+          row[indexer["attacks"]] if row[indexer["attacks"]][0] != '-' else None, # attacks, formatted
+          row[indexer["leadership"]] if row[indexer["leadership"]][0] != '-' else None, # leadership, formatted
+          row[indexer["save"]][0] if row[indexer["save"]][0] != '-' else None, # save, deformatted
+          row[indexer["cost"]], # cost
+          row[indexer["cost_description"]], # cost_description
+          row[indexer["models_per_unit"]], # models_per_unit
+          1 if row[indexer["cost_including_wargear"]] == "true" else 0 if row[indexer["cost_including_wargear"]] == "false" else -1, # cost_including_wargear, boolean
+          row[indexer["base_size"]], # base_size
+          row[indexer["base_size_descr"]], # base_size_descr
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into datasheets_models")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Datasheets_models.csv")
+  
+  print("\nDATASHEETS OPTIONS")
+  try:
+    with open("wahapedia/Datasheets_options.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "datasheet_id"):
+          indexer["datasheet_id"] = i
+        elif(column == "line"):
+          indexer["line"] = i
+        elif(column == "button"):
+          indexer["button"] = i
+        elif(column == "description"):
+          indexer["description"] = i
+        elif(column == "is_index_wargear"):
+          indexer["is_index_wargear"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO datasheets_options (datasheet_id, line, button, description, is_index_wargear) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s),"
+        data = data + [
+          row[indexer["datasheet_id"]], # datasheet_id
+          row[indexer["line"]], # line
+          row[indexer["button"]], # button
+          row[indexer["description"]], # description
+          1 if row[indexer["is_index_wargear"]] == "true" else 0 if row[indexer["is_index_wargear"]] == "false" else -1, # is_index_wargear, boolean
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into datasheets_options")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Datasheets_options.csv")
+  
+  print("\nDATASHEETS WARGEAR")
+  try:
+    with open("wahapedia/Datasheets_wargear.csv",mode='r') as f:
+      csvF = csv.reader(f, delimiter='|')
+      
+      indexer = {}
+      i = 0
+      columns = next(csvF)
+      for column in columns:
+        column = column.replace(u'\ufeff', '')
+        if(column == "datasheet_id"):
+          indexer["datasheet_id"] = i
+        elif(column == "line"):
+          indexer["line"] = i
+        elif(column == "wargear_id"):
+          indexer["wargear_id"] = i
+        elif(column == "cost"):
+          indexer["cost"] = i
+        elif(column == "is_index_wargear"):
+          indexer["is_index_wargear"] = i
+        elif(column == "model"):
+          indexer["model"] = i
+        elif(column == "is_upgrade"):
+          indexer["is_upgrade"] = i
+        i = i + 1
+      
+      query = "INSERT IGNORE INTO datasheets_wargear (datasheet_id, line, wargear_id, cost, is_index_wargear, model, is_upgrade) VALUES "
+      data = []
+      for row in csvF:
+        query+="(%s,%s,%s,%s,%s,%s,%s),"
+        # MySQL doesn't support booleans, so we're re-using an INT (-1 is data isn't boolean)
+        data = data + [
+          row[indexer["datasheet_id"]], # datasheet_id
+          row[indexer["line"]], # line
+          row[indexer["wargear_id"]], # wargear_id
+          row[indexer["cost"]], # cost
+          1 if row[indexer["is_index_wargear"]] == "true" else 0 if row[indexer["is_index_wargear"]] == "false" else -1, # is_index_wargear, boolean
+          row[indexer["model"]], # model
+          1 if row[indexer["is_upgrade"]] == "true" else 0 if row[indexer["is_upgrade"]] == "false" else -1, # is_upgrade, boolean
+        ]
+      query = query[:-1] + " ON DUPLICATE KEY UPDATE check_me=1"
+      cur.execute(query,data)
+      print(cur.rowcount, "rows inserted into datasheets_wargear")
+      db.commit()
+  except mysql.connector.Error as err:
+    print(err)
+  except IOError:
+    print("unable to open wahapedia/Datasheets_wargear.csv")
+  
+  print("\nDATA ERRORS")
+  try:
+    query = "SELECT COUNT(*) AS count FROM sources WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in sources".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM factions WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in factions".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM wargear WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in wargear".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM wargear_list WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in wargear_list".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM datasheets WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in datasheets".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM abilities WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in abilities".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM datasheets_abilities WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in datasheets_abilities".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM psychic_powers WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in psychic_powers".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM warlord_traits WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in warlord_traits".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM strategems WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in strategems".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM datasheets_damage WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in datasheets_damage".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM datasheets_keywords WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in datasheets_keywords".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM datasheets_models WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in datasheets_models".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM datasheets_options WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in datasheets_options".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  try:
+    query = "SELECT COUNT(*) AS count FROM datasheets_wargear WHERE check_me = 1"
+    cur.execute(query)
+    errors = cur.fetchone().count
+    if(errors):
+      print("{count} Error(s) in datasheets_wargear".format(count=errors))
+  except mysql.connector.Error as err:
+    print(err)
+  
+  print("\nscript ending")
+  cur.close()
+  db.close()

wahapedia_import/package-lock.json

@@ -0,0 +1,3 @@
+{
+  "lockfileVersion": 1
+}

wahapedia_import/updateInputData.py

@@ -0,0 +1,82 @@
+import requests
+import os
+
+if(not os.path.exists("./wahapedia")):
+  os.mkdir("./wahapedia")
+
+print("getting Abilities")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Abilities.csv")
+with open("./wahapedia/Abilities.csv", "wb") as f:
+  f.write(r.content)
+
+print("getting Datasheets")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Datasheets.csv")
+with open("./wahapedia/Datasheets.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Datasheets_abilities")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Datasheets_abilities.csv")
+with open("./wahapedia/Datasheets_abilities.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Datasheets_damage")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Datasheets_damage.csv")
+with open("./wahapedia/Datasheets_damage.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Datasheets_keywords")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Datasheets_keywords.csv")
+with open("./wahapedia/Datasheets_keywords.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Datasheets_models")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Datasheets_models.csv")
+with open("./wahapedia/Datasheets_models.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Datasheets_options")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Datasheets_options.csv")
+with open("./wahapedia/Datasheets_options.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Datasheets_wargear")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Datasheets_wargear.csv")
+with open("./wahapedia/Datasheets_wargear.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Factions")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Factions.csv")
+with open("./wahapedia/Factions.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting PsychicPowers")
+r = requests.get("https://wahapedia.ru/wh40k9ed/PsychicPowers.csv")
+with open("./wahapedia/PsychicPowers.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Source")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Source.csv")
+with open("./wahapedia/Source.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Stratagems")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Stratagems.csv")
+with open("./wahapedia/Stratagems.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Wargear")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Wargear.csv")
+with open("./wahapedia/Wargear.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Wargear_list")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Wargear_list.csv")
+with open("./wahapedia/Wargear_list.csv", "wb") as f:
+  f.write(r.content)
+  
+print("getting Warlord_traits")
+r = requests.get("https://wahapedia.ru/wh40k9ed/Warlord_traits.csv")
+with open("./wahapedia/Warlord_traits.csv", "wb") as f:
+  f.write(r.content)
+
+print("done downloading files")

wahapedia_import/wahapediaSetupDatabase.sql

@@ -0,0 +1,241 @@
+START TRANSACTION;
+
+DROP TABLE IF EXISTS datasheets_wargear;
+DROP TABLE IF EXISTS datasheets_options;
+DROP TABLE IF EXISTS datasheets_models;
+DROP TABLE IF EXISTS datasheets_keywords;
+DROP TABLE IF EXISTS datasheets_damage;
+DROP TABLE IF EXISTS strategems;
+DROP TABLE IF EXISTS warlord_traits;
+DROP TABLE IF EXISTS psychic_powers;
+DROP TABLE IF EXISTS datasheets_abilities;
+DROP TABLE IF EXISTS abilities;
+DROP TABLE IF EXISTS datasheets;
+DROP TABLE IF EXISTS wargear_list;
+DROP TABLE IF EXISTS wargear;
+DROP TABLE IF EXISTS factions;
+DROP TABLE IF EXISTS sources;
+
+ALTER DATABASE wahapedia CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "sources";
+CREATE TABLE sources (
+  source_id INT PRIMARY KEY,
+  name VARCHAR(255),
+  type VARCHAR(255),
+  edition INT,
+  version VARCHAR(255),
+  errata_date VARCHAR(255),
+  errata_link VARCHAR(255),
+  check_me INT DEFAULT 0 /*boolean*/
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "factions";
+CREATE TABLE factions (
+  faction_id VARCHAR(255) PRIMARY KEY,
+  name VARCHAR(255),
+  link VARCHAR(255),
+  is_subfaction VARCHAR(255),
+  main_faction_id VARCHAR(255),
+  check_me INT DEFAULT 0 /*boolean*/
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "wargear";
+CREATE TABLE wargear (
+  wargear_id INT PRIMARY KEY,
+  name VARCHAR(255),
+  type VARCHAR(255),
+  description TEXT,
+  is_relic INT, /*boolean*/
+  faction_id VARCHAR(255),
+  legend TEXT,
+  check_me INT DEFAULT 0, /*boolean*/
+  CONSTRAINT wargear_fk_factions FOREIGN KEY (faction_id) REFERENCES factions (faction_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "wargear_list";
+CREATE TABLE wargear_list (
+  wargear_id INT,
+  line  INT,
+  name VARCHAR(255),
+  weapon_range VARCHAR(255),
+  type VARCHAR(255),
+  strength VARCHAR(255),
+  armor_piercing VARCHAR(255),
+  damage VARCHAR(255),
+  abilities TEXT,
+  check_me INT DEFAULT 0, /*boolean*/
+  PRIMARY KEY (wargear_id, line),
+  CONSTRAINT wargear_list_fk_wargear FOREIGN KEY (wargear_id) REFERENCES wargear (wargear_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "datasheets";
+CREATE TABLE datasheets (
+  datasheet_id INT PRIMARY KEY,
+  name VARCHAR(255),
+  link VARCHAR(255),
+  faction_id VARCHAR(255),
+  source_id INT,
+  role VARCHAR(255),
+  unit_composition TEXT,
+  transport TEXT,
+  power_points VARCHAR(255),
+  priest TEXT,
+  psyker TEXT,
+  open_play_only INT, /*boolean*/
+  crusade_only INT, /*boolean*/
+  virtual_ INT, /*boolean*/
+  cost INT,
+  cost_per_unit INT, /*boolean*/
+  check_me INT DEFAULT 0, /*boolean*/
+  CONSTRAINT datasheets_fk_factions FOREIGN KEY (faction_id) REFERENCES factions (faction_id),
+  CONSTRAINT datasheets_fk_sources FOREIGN KEY (source_id) REFERENCES sources (source_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "abilities";
+CREATE TABLE abilities (
+  ability_id INT PRIMARY KEY,
+  type VARCHAR(255),
+  name VARCHAR(255),
+  legend TEXT,
+  is_other_wargear INT, /*boolean*/
+  faction_id VARCHAR(255),
+  description TEXT,
+  check_me INT DEFAULT 0, /*boolean*/
+  CONSTRAINT abilities_fk_factions FOREIGN KEY (faction_id) REFERENCES factions (faction_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "datasheets_abilities";
+CREATE TABLE datasheets_abilities (
+  datasheet_id INT,
+  line INT,
+  ability_id INT,
+  is_index_wargear INT, /*boolean*/
+  cost INT,
+  model VARCHAR(255),
+  check_me INT DEFAULT 0, /*boolean*/
+  PRIMARY KEY (datasheet_id, line),
+  CONSTRAINT datasheets_abilities_fk_datasheets FOREIGN KEY (datasheet_id) REFERENCES datasheets (datasheet_id),
+  CONSTRAINT datasheets_abilities_fk_abilities FOREIGN KEY (ability_id) REFERENCES abilities (ability_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "psychic_powers";
+CREATE TABLE psychic_powers (
+  psychic_power_id INT,
+  roll INT,
+  name VARCHAR(255),
+  faction_id VARCHAR(255),
+  legend TEXT,
+  type VARCHAR(255),
+  description TEXT,
+  check_me INT DEFAULT 0, /*boolean*/
+  PRIMARY KEY (psychic_power_id, roll),
+  CONSTRAINT psychic_powers_fk_factions FOREIGN KEY (faction_id) REFERENCES factions (faction_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "warlord_traits";
+CREATE TABLE warlord_traits (
+  trait_id INT PRIMARY KEY AUTO_INCREMENT,
+  faction_id VARCHAR(255),
+  type VARCHAR(255),
+  roll VARCHAR(255),
+  name VARCHAR(255),
+  legend TEXT,
+  description TEXT,
+  check_me INT DEFAULT 0, /*boolean*/
+  CONSTRAINT warlord_traits_fk_factions FOREIGN KEY (faction_id) REFERENCES factions (faction_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "strategems";
+CREATE TABLE strategems (
+  strategem_id INT PRIMARY KEY,
+  faction_id VARCHAR(255),
+  name VARCHAR(255),
+  type VARCHAR(255),
+  cp_cost VARCHAR(255),
+  legend TEXT,
+  source_id INT,
+  description TEXT,
+  check_me INT DEFAULT 0, /*boolean*/
+  CONSTRAINT strategems_fk_factions FOREIGN KEY (faction_id) REFERENCES factions (faction_id),
+  CONSTRAINT strategems_fk_sources FOREIGN KEY (source_id) REFERENCES sources (source_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "datasheets_damage";
+CREATE TABLE datasheets_damage (
+  datasheet_id INT,
+  line INT,
+  col1 VARCHAR(255),
+  col2 VARCHAR(255),
+  col3 VARCHAR(255),
+  col4 VARCHAR(255),
+  col5 VARCHAR(255),
+  check_me INT DEFAULT 0, /*boolean*/
+  PRIMARY KEY (datasheet_id, line),
+  CONSTRAINT datasheets_damage_fk_datasheets FOREIGN KEY (datasheet_id) REFERENCES datasheets (datasheet_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "datasheets_keywords";
+CREATE TABLE datasheets_keywords (
+  datasheet_id INT,
+  keyword VARCHAR(255),
+  model VARCHAR(255),
+  is_faction_keyword INT, /*boolean*/
+  check_me INT DEFAULT 0, /*boolean*/
+  PRIMARY KEY (datasheet_id, keyword),
+  CONSTRAINT datasheets_keywords_fk_datasheets FOREIGN KEY (datasheet_id) REFERENCES datasheets (datasheet_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "datasheets_models";
+CREATE TABLE datasheets_models (
+  datasheet_id INT,
+  line INT,
+  name VARCHAR(255),
+  movement VARCHAR(255),
+  weapon_skill INT,
+  ballistic_skill INT,
+  strength INT,
+  toughness INT,
+  wounds INT,
+  attacks VARCHAR(255),
+  leadership INT,
+  save INT,
+  cost INT,
+  cost_description TEXT,
+  models_per_unit VARCHAR(255),
+  cost_including_wargear INT, /*boolean*/
+  base_size VARCHAR(255),
+  base_size_descr VARCHAR(255),
+  check_me INT DEFAULT 0, /*boolean*/
+  PRIMARY KEY (datasheet_id, line),
+  CONSTRAINT datasheets_models_fk_datasheets FOREIGN KEY (datasheet_id) REFERENCES datasheets (datasheet_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "datasheets_options";
+CREATE TABLE datasheets_options (
+  datasheet_id INT,
+  line INT,
+  button VARCHAR(255),
+  description TEXT,
+  is_index_wargear INT, /*boolean*/
+  check_me INT DEFAULT 0, /*boolean*/
+  PRIMARY KEY (datasheet_id, line),
+  CONSTRAINT datasheets_options_fk_datasheets FOREIGN KEY (datasheet_id) REFERENCES datasheets (datasheet_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+SELECT "datasheets_wargear";
+CREATE TABLE datasheets_wargear (
+  datasheet_id INT,
+  line INT,
+  wargear_id INT,
+  cost INT,
+  is_index_wargear INT, /*boolean*/
+  model VARCHAR(255),
+  is_upgrade INT, /*boolean*/
+  check_me INT DEFAULT 0, /*boolean*/
+  PRIMARY KEY (datasheet_id, line, is_upgrade),
+  CONSTRAINT datasheets_wargear_fk_datasheets FOREIGN KEY (datasheet_id) REFERENCES datasheets (datasheet_id),
+  CONSTRAINT datasheets_wargear_fk_wargear FOREIGN KEY (wargear_id) REFERENCES wargear (wargear_id)
+) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+COMMIT;