datajams-lbai / datajams-evictions (public) (License: AGPLv3) (since 2020-09-01) (hash sha1)
Extracting Harris County evictions information for Houston DataJams
Clone URLs: https://rocketgit.com/user/datajams-lbai/datajams-evictions ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions git://git.rocketgit.com/user/datajams-lbai/datajams-evictions
first
List of commits:
Subject Hash Author Date (UTC)
Clean up data xfer. Use TRUNCATE TABLE instead of DELETE FROM. e407788ccd1fb9ace1ca7cad4b0a1acd0189c799 Luigi Bai 2020-09-16 02:37:15
Clean up code. 3036e8ca177024179b3a6e391876121e526cd213 Luigi Bai 2020-09-16 02:36:42
Add bulkExtract importer. 0da4130e3e3a3e2e855f2ef28c4194fb6b5dfe10 Luigi Bai 2020-09-16 02:36:02
Add court related data 20f13f9ef9885794eddb2470bd57a9207dc7af20 Luigi Bai 2020-09-12 21:47:34
Try to avoid errors where information may be missing. 3410e49ba53bd7ebb417ee9bac7db1cf9e0e8808 Luigi Bai 2020-09-09 17:08:59
Don't put status information into console.log. 19dab8ff60a0256f9511391d4088c1b8f88ac3db Luigi Bai 2020-09-09 17:06:24
Add more info to README. 9708b396b887bab284e0ad2c2d996ea2db0ff4d4 Luigi Bai 2020-09-09 12:37:24
Add info to README. 79672fad423a570d7eb05c4891dfabb511db5ab3 Luigi Bai 2020-09-09 00:20:23
Fixed searchNames. Now you can load cases where you know one of the parties. 0ad812704e27341a481a6de18c88a4f38bb61700 Luigi Bai 2020-09-08 23:43:48
Apparently updating addresses works, as long as you don't ask for more than 10 per minute. That seems to be working so far. 9276b2acb2cc3067c53fc16317a8960db3f6fcca Luigi Bai 2020-09-08 22:44:24
More information about how to use the software. Not much, yet. Enough to get started, I think. 393fce58bbcfeaee947cb2b0e17d81d16da49612 Luigi Bai 2020-09-08 22:31:02
Don't write to console.err if there's a rejectionFunction. 630b8ec76a21733a6bb93d98ca54563aadd61da4 Luigi Bai 2020-09-08 22:30:29
Remove trailing comma on all rows. e7605fa152a841eefc7fcab6bb7cfd64daf28b17 Luigi Bai 2020-09-07 22:43:36
Allow passing in query for Case.sqlAllCases() 346862c0720ef9054426048cc95caa8f6b068868 Luigi Bai 2020-09-07 22:43:09
This version works in MS-SQL. 4e3f33f464173c91c3f4ac4b6acb0235ed2d6720 Luigi Bai 2020-09-07 19:11:54
Add headers. 4da1ccbdb2e3e852050de3460bc9ac9ba061ed7a Luigi Bai 2020-09-07 16:28:52
loadDockets now works to local. 96ba8f7419f63be6612fe1efd3e24d5d30eaf20a Luigi Bai 2020-09-06 20:59:06
Fix for new column names. 5791d77e19585c494e51c6062ab45440e5af0903 Luigi Bai 2020-09-06 20:44:50
Add headers 32cb7264c06b28dc0404b715f24fd2d82b66633b Luigi Bai 2020-09-06 20:44:29
Have a working version of xfer - now transfers data to MS SQL Server from SQLITE3. 428e493bcc5b7f718a741db046d08905f47d0f1c Luigi Bai 2020-09-05 22:11:55
Commit e407788ccd1fb9ace1ca7cad4b0a1acd0189c799 - Clean up data xfer. Use TRUNCATE TABLE instead of DELETE FROM.
Author: Luigi Bai
Author date (UTC): 2020-09-16 02:37
Committer name: Luigi Bai
Committer date (UTC): 2020-09-16 02:37
Parent(s): 3036e8ca177024179b3a6e391876121e526cd213
Signing key:
Tree: 5be59d8d058b764f7d3c7759299e667f7d2a0a21
File Lines added Lines deleted
xferData.js 14 11
File xferData.js changed (mode: 100644) (index 67a852a..dba4385)
... ... class xferCtlr {
46 46 return new Promise(async (resFN, rejFN) => { return new Promise(async (resFN, rejFN) => {
47 47 try { try {
48 48 let msReq = new msD.Request(); let msReq = new msD.Request();
49 let res = await msReq.batch("DELETE FROM "+this.tableName);
49 let res = await msReq.batch("TRUNCATE TABLE "+this.tableName);
50 50 await msReq.cancel(); await msReq.cancel();
51 51 resFN({ msg: "DELETE FROM "+this.tableName+" done", val: res }); resFN({ msg: "DELETE FROM "+this.tableName+" done", val: res });
52 52 } catch (e) { } catch (e) {
 
... ... class xferCtlr {
64 64 resFN(count); resFN(count);
65 65 } else { } else {
66 66 destS.execute(row, (err, res) => { destS.execute(row, (err, res) => {
67 if (err) {
68 if (this.ignoreInsertErrors)
69 console.error(err);
70 else
71 rejFN({ e: err, n: count });
67 if (err && !this.ignoreInsertErrors) {
68 rejFN({ e: err, n: count });
72 69 } else { } else {
70 if (err) console.error(err);
73 71 (count % 100 === 0) && console.log(this.tableName+" Count: "+count); (count % 100 === 0) && console.log(this.tableName+" Count: "+count);
74 72 this._oneRow(srcS, destS, resFN, rejFN, count + 1); this._oneRow(srcS, destS, resFN, rejFN, count + 1);
75 73 } }
 
... ... class xferCtlr {
89 87 .then(() => { .then(() => {
90 88 this._oneRow( this._oneRow(
91 89 selStmt, insStmt, selStmt, insStmt,
92 async (val) => {
90 async val => {
93 91 this.sinkCount = val; this.sinkCount = val;
94 92 selStmt.finalize(); selStmt.finalize();
95 93 await insStmt.unprepare(); await insStmt.unprepare();
 
... ... class xferCtlr {
100 98 sinkRows: this.sinkCount sinkRows: this.sinkCount
101 99 }); });
102 100 }, },
103 async (errO) => {
101 async errO => {
104 102 this.sinkCount = errO.n; this.sinkCount = errO.n;
105 103 selStmt.finalize(); selStmt.finalize();
106 104 await insStmt.unprepare(); await insStmt.unprepare();
 
... ... class xferCtlr {
137 135
138 136 let casesCtlr = new xferCtlr({ let casesCtlr = new xferCtlr({
139 137 tableName: "cases", tableName: "cases",
140 sqlRowSEL: "SELECT casenumber cnum, case_URL curl, filed_date fdt, case_status cstat FROM cases",
141 sqlRowINS: "INSERT INTO cases (casenumber, case_URL, filed_date, case_status) VALUES (@cnum, @curl, @fdt, @cstat)",
138 sqlRowSEL: "SELECT casenumber cnum, case_URL curl, filed_date fdt, case_status cstat, odyssey_ID oi FROM cases",
139 sqlRowINS: "INSERT INTO cases (casenumber, case_URL, filed_date, case_status, odyssey_ID) VALUES (@cnum, @curl, @fdt, @cstat, @oi)",
142 140 insBind: (db, stmt) => { insBind: (db, stmt) => {
143 141 stmt.input("cnum", db.NVarChar); stmt.input("cnum", db.NVarChar);
144 142 stmt.input("curl", db.NVarChar); stmt.input("curl", db.NVarChar);
145 143 stmt.input("fdt", db.NVarChar); stmt.input("fdt", db.NVarChar);
146 144 stmt.input("cstat", db.NVarChar); stmt.input("cstat", db.NVarChar);
145 stmt.input("oi", db.NVarChar);
147 146 } }
148 147 }); });
149 148 let docketsCtlr = new xferCtlr({ let docketsCtlr = new xferCtlr({
 
... ... opts.connectCallback = (sqDB) => {
198 197 let opts = require("./creds")["MS"]; let opts = require("./creds")["MS"];
199 198 opts.connectCallback = async (msDB) => { opts.connectCallback = async (msDB) => {
200 199 try { try {
200 casesCtlr.ignoreInsertErrors = true;
201 201 console.log(await casesCtlr.xferPromise(sqDB, msDB)); console.log(await casesCtlr.xferPromise(sqDB, msDB));
202 docketsCtlr.ignoreInsertErrors = true;
202 203 console.log(await docketsCtlr.xferPromise(sqDB, msDB)); console.log(await docketsCtlr.xferPromise(sqDB, msDB));
204 dcCtlr.ignoreInsertErrors = true;
203 205 console.log(await dcCtlr.xferPromise(sqDB, msDB)); console.log(await dcCtlr.xferPromise(sqDB, msDB));
204 // partyCtlr.ignoreInsertErrors = true;
206 partyCtlr.ignoreInsertErrors = true;
205 207 console.log(await partyCtlr.xferPromise(sqDB, msDB)); console.log(await partyCtlr.xferPromise(sqDB, msDB));
208 eventsCtlr.ignoreInsertErrors = true;
206 209 console.log(await eventsCtlr.xferPromise(sqDB, msDB)); console.log(await eventsCtlr.xferPromise(sqDB, msDB));
207 210 } catch (e) { } catch (e) {
208 211 console.error(e); console.error(e);
Hints:
Before first commit, do not forget to setup your git environment:
git config --global user.name "your_name_here"
git config --global user.email "your@email_here"

Clone this repository using HTTP(S):
git clone https://rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using ssh (do not forget to upload a key first):
git clone ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using git:
git clone git://git.rocketgit.com/user/datajams-lbai/datajams-evictions

You are allowed to anonymously push to this repository.
This means that your pushed commits will automatically be transformed into a merge request:
... clone the repository ...
... make some changes and some commits ...
git push origin main