datajams-lbai / datajams-evictions (public) (License: AGPLv3) (since 2020-09-01) (hash sha1)
Extracting Harris County evictions information for Houston DataJams
Clone URLs: https://rocketgit.com/user/datajams-lbai/datajams-evictions ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions git://git.rocketgit.com/user/datajams-lbai/datajams-evictions
first
List of commits:
Subject Hash Author Date (UTC)
Transfer local data to cloud based MS SQL-Server database. 8035b6dac5ce05ced62474409ab46ca490a6db4a Luigi Bai 2020-09-04 03:50:32
Learned that package-lock.json should also be committed. 20222232bd2e5388b76d3330e6f695b5aacfe393 Luigi Bai 2020-09-03 11:04:05
Fixed SPDX license expression. c894ba842750ff63035c850095ac93399495f4bc Luigi Bai 2020-09-03 11:01:27
Updated README. Added .gitignore, node/npm package.json, and the schema definitions for the data stores. a5baa30d1360514e217250737abf3a4518ab6a65 Luigi Bai 2020-09-02 17:35:09
Initial commit f67ef5b180d66c95e84ee5c5aa0a893376afa5c4 datajams-lbai 2020-09-02 02:28:52
Commit 8035b6dac5ce05ced62474409ab46ca490a6db4a - Transfer local data to cloud based MS SQL-Server database.
Author: Luigi Bai
Author date (UTC): 2020-09-04 03:50
Committer name: Luigi Bai
Committer date (UTC): 2020-09-04 03:50
Parent(s): 20222232bd2e5388b76d3330e6f695b5aacfe393
Signer:
Signing key:
Signing status: N
Tree: 8c1e687a04d997e3cea72703511198c2042007df
File Lines added Lines deleted
lib/msDAO.js 36 0
lib/sqlDAO.js 128 0
xferData.js 150 0
File lib/msDAO.js added (mode: 100644) (index 0000000..b3ec8e0)
1 const sql = require("mssql");
2
3 let DAO = class {
4 constructor(opts) {
5 };
6 };
7
8 module.exports = {
9 connect(opts) {
10 if (!opts) throw "Must set credentials and callback";
11 if (!opts.user) throw "Must set user in opts";
12 if (!opts.pass) throw "Must set pass in opts";
13
14 let config = {
15 user: opts.user,
16 password: opts.pass,
17 server: "harriscountyhousing.database.windows.net",
18 database: "DataJams_HarrisCountyHousing",
19 encrypt: true,
20 options: {
21 enableArithAbort: true
22 }
23 };
24
25 sql.connect(config)
26 .then(() => {
27 opts.connectCallback && "function" === typeof opts.connectCallback && opts.connectCallback(sql);
28 })
29 .catch(err => { console.log("Connect Error: ", err); })
30 ;
31 },
32
33 disconnect(database) {
34 database.close().catch(err => { console.log("Close Error: ", err); });
35 }
36 };
File lib/sqlDAO.js added (mode: 100644) (index 0000000..ad211b8)
1 const sq3 = require("sqlite3");
2
3 let DAO = class {
4
5 constructor() {
6 this.stmts = {};
7 };
8
9 reportErr(e) {
10 console.error(typeof this.dao, e);
11 };
12
13 prepare(database, sql) {
14 let stmt = database.prepare(sql);
15 stmt.dao = this;
16 return stmt;
17 };
18
19 insert(dObj) {
20 this.dataObj = dObj;
21 if (this.stmts.insert) {
22 this.stmts.insert.run(dObj, this.errInsert);
23 } else {
24 this.reportErr({ error: "No INSERT statement?" });
25 }
26 };
27 errInsert(e) {
28 if (e) {
29 switch (e.errno) {
30 case 19:
31 if (this.dao.stmts.update) {
32 console.error("I would do an update here: ", this.dao.dataObj);
33 } else {
34 // In general, ignore the conflict:
35 }
36 break;
37 default:
38 this.dao.reportErr(e);
39 break;
40 }
41 }
42 };
43
44 update(dObj) {
45 this.dataObj = dObj;
46 if (this.stmts.update) {
47 this.stmts.update.run(dObj, this.errUpdate);
48 } else {
49 this.reportErr({ error: "No UPDATE statement?" });
50 }
51 };
52 errUpdate(e) {
53 if (e) {
54 switch (e.errno) {
55 default:
56 this.dao.reportErr(e);
57 break;
58 }
59 }
60 };
61 };
62
63 module.exports = {
64 DAO: DAO,
65
66 connect(opts) {
67 sq3.verbose();
68 let database = new sq3.Database("eviction-data.sq3", (err) => {
69 if (err) {
70 throw err;
71 }
72 });
73 opts && "function" === typeof opts.connectCallback && opts.connectCallback(database);
74 },
75
76 disconnect(db) {
77 db.close();
78 },
79
80 DocketCtlr: class extends DAO {
81 constructor (database) {
82 super();
83 this.stmts.insert = this.prepare(
84 database,
85 "INSERT INTO docket (precinct, place, dateTime, URL) \
86 VALUES ($pct, $plc, $dt, $url)"
87 );
88 };
89
90 },
91 DocketCaseCtlr: class extends DAO {
92 constructor(database) {
93 super();
94 this.stmts.insert = this.prepare(
95 database,
96 "INSERT INTO docketedCases (precinct, place, dateTIme, casenumber, claim) VALUES ($pct, $plc, $dt, $cnum, $claim)"
97 );
98 };
99 },
100 CaseCtlr: class extends DAO {
101 constructor(database) {
102 super();
103 this.stmts.insert = this.prepare(
104 database,
105 "INSERT INTO cases (casenumber, caseURL) VALUES ($cnum, $curl)"
106 );
107 };
108 },
109 PartyCtlr: class extends DAO {
110 constructor(database) {
111 super();
112 this.stmts.insert = this.prepare(
113 database,
114 "INSERT INTO party (casenumber, role, name) VALUES ($cnum, $role, $name)"
115 );
116 };
117 },
118 EventCtlr: class extends DAO {
119 constructor(database) {
120 super();
121 this.stmts.insert = this.prepare(
122 database,
123 "INSERT INTO events (casenumber, description, dateAdded) VALUES ($cnum, $desc, $date)"
124 );
125 };
126 }
127 };
128
File xferData.js added (mode: 100644) (index 0000000..27eddf8)
1 const sqDAO = require("./lib/sqlDAO");
2 const msDAO = require("./lib/msDAO");
3
4 class xferCtlr {
5 constructor(props) {
6 Object.assign(this, props);
7 };
8
9 xferPromise(sqD, msD) {
10 return new Promise(async (resFN, rejFN) => {
11 sqD.each(
12 "SELECT COUNT(*) num FROM "+this.tableName,
13 (err, row) => {
14 if (err) {
15 rejFN(err);
16 } else {
17 console.log(this.tableName, "SOURCE rows: ", row);
18 }
19 }
20 );
21
22 msD.batch("DELETE FROM "+this.tableName)
23 .then(result => {
24 let oneRow = (sel, ins, num) => {
25 sel.get((err, row) => {
26 if (err) {
27 rejFN(err);
28 } else {
29 if (row) {
30 ins.execute(row)
31 .then(() => {
32 oneRow(sel, ins, num+1);
33 })
34 .catch(e => {
35 sel.finalize();
36 ins.unprepare()
37 .finally(() => { rejFN(e); })
38 ;
39 })
40 ;
41 } else {
42 sel.finalize();
43 ins.unprepare()
44 .catch(e => { rejFN(e); });
45 resFN({ tbl: this.tableName, msg: "Finished.", num: num });
46 }
47 }
48 });
49 };
50
51 let selStmt = sqD.prepare(this.sqlRowSEL);
52 let insStmt = new msD.PreparedStatement();
53 this.insBind(msD, insStmt);
54 insStmt.prepare(this.sqlRowINS)
55 .then(() => {
56 oneRow(selStmt, insStmt, 0);
57 })
58 .catch(err => { selStmt.finalize(); rejFN(err); })
59 ;
60 })
61 .catch(e => {
62 rejFN(e);
63 })
64 ;
65 });
66 };
67 };
68
69
70 opts.connectCallback = (sqDB) => {
71 // Now get the msSQL connection:
72 let opts = require("./creds")["MS"];
73 opts.connectCallback = (msDB) => {
74 let casesCtlr = new xferCtlr({
75 tableName: "CASES",
76 sqlRowSEL: "SELECT casenumber cnum, case_URL curl, filed_date fdt, case_status cstat FROM CASES",
77 sqlRowINS: "INSERT INTO CASES (casenumber, case_URL, filed_date, case_status) VALUES (@cnum, @curl, @fdt, @cstat)",
78 insBind: (db, stmt) => {
79 stmt.input("cnum", db.NVarChar);
80 stmt.input("curl", db.NVarChar);
81 stmt.input("fdt", db.NVarChar);
82 stmt.input("cstat", db.NVarChar);
83 }
84 });
85 let docketsCtlr = new xferCtlr({
86 tableName: "DOCKET",
87 sqlRowSEL: "SELECT precinct pct, place plc, docket_dateTime ddt, URL url FROM DOCKET",
88 sqlRowINS: "INSERT INTO DOCKET (precinct, place, docket_dateTime, URL) VALUES (@pct, @plc, @ddt, @url)",
89 insBind: (db, stmt) => {
90 stmt.input("pct", db.Int);
91 stmt.input("plc", db.Int);
92 stmt.input("ddt", db.NVarChar);
93 stmt.input("url", db.NVarChar);
94 }
95 });
96 let dcCtlr = new xferCtlr({
97 tableName: "DOCKETEDCASES",
98 sqlRowSEL: "SELECT precinct pct, place plc, docket_dateTime ddt, casenumber cnum, claim clm FROM DOCKETEDCASES",
99 sqlRowINS: "INSERT INTO DOCKETEDCASES (precinct, place, docket_dateTime, casenumber, claim) VALUES (@pct, @plc, @ddt, @cnum, @clm)",
100 insBind: (db, stmt) => {
101 stmt.input("pct", db.Int);
102 stmt.input("plc", db.Int);
103 stmt.input("ddt", db.NVarChar);
104 stmt.input("cnum", db.NVarChar);
105 stmt.input("clm", db.NVarChar);
106 }
107 });
108 let partyCtlr = new xferCtlr({
109 tableName: "PARTY",
110 sqlRowSEL: "SELECT casenumber cnum, party_role pr, party_name pn, party_address pa FROM PARTY",
111 sqlRowINS: "INSERT INTO PARTY (casenumber, party_role, party_name, party_address) VALUES (@cnum, @pr, @pn, @pa)",
112 insBind: (db, stmt) => {
113 stmt.input("pr", db.NVarChar);
114 stmt.input("pn", db.NVarChar);
115 stmt.input("pa", db.NVarChar);
116 stmt.input("cnum", db.NVarChar);
117 }
118 });
119 let eventsCtlr = new xferCtlr({
120 tableName: "EVENTS",
121 sqlRowSEL: "SELECT casenumber cnum, eventDescription ed, dateAdded da FROM EVENTS",
122 sqlRowINS: "INSERT INTO EVENTS (casenumber, eventDescription, dateAdded) VALUES (@cnum, @ed, @da)",
123 insBind: (db, stmt) => {
124 stmt.input("cnum", db.NVarChar);
125 stmt.input("ed", db.NVarChar);
126 stmt.input("da", db.NVarChar);
127 }
128 });
129
130 Promise.all([
131 casesCtlr.xferPromise(sqDB, msDB).catch(e => { console.log("CTLR CALLBACK: ", e); }),
132 docketsCtlr.xferPromise(sqDB, msDB).catch(e => { console.log("CTLR CALLBACK: ", e); }),
133 dcCtlr.xferPromise(sqDB, msDB).catch(e => { console.log("CTLR CALLBACK: ", e); }),
134 partyCtlr.xferPromise(sqDB, msDB).catch(e => { console.log("CTLR CALLBACK: ", e); }),
135 eventsCtlr.xferPromise(sqDB, msDB).catch(e => { console.log("CTLR CALLBACK: ", e); }),
136 ])
137 .then(resArr => { resArr.forEach(val => { val && console.log(val); }); })
138 .catch(e => { console.log("ALL CALLBACK: ", e); })
139 .finally(() => {
140 msDAO.disconnect(msDB);
141 sqDAO.disconnect(sqDB);
142 })
143 ;
144 };
145
146 msDAO.connect(opts);
147 };
148
149 let opts = require("./creds")["SQLITE3"];
150 sqDAO.connect(opts);
Hints:
Before first commit, do not forget to setup your git environment:
git config --global user.name "your_name_here"
git config --global user.email "your@email_here"

Clone this repository using HTTP(S):
git clone https://rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using ssh (do not forget to upload a key first):
git clone ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using git:
git clone git://git.rocketgit.com/user/datajams-lbai/datajams-evictions

You are allowed to anonymously push to this repository.
This means that your pushed commits will automatically be transformed into a merge request:
... clone the repository ...
... make some changes and some commits ...
git push origin main