datajams-lbai / datajams-evictions (public) (License: AGPLv3) (since 2020-09-01) (hash sha1)
Extracting Harris County evictions information for Houston DataJams
Clone URLs: https://rocketgit.com/user/datajams-lbai/datajams-evictions ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions git://git.rocketgit.com/user/datajams-lbai/datajams-evictions
first
List of commits:
Subject Hash Author Date (UTC)
Clean up code. 3036e8ca177024179b3a6e391876121e526cd213 Luigi Bai 2020-09-16 02:36:42
Add bulkExtract importer. 0da4130e3e3a3e2e855f2ef28c4194fb6b5dfe10 Luigi Bai 2020-09-16 02:36:02
Add court related data 20f13f9ef9885794eddb2470bd57a9207dc7af20 Luigi Bai 2020-09-12 21:47:34
Try to avoid errors where information may be missing. 3410e49ba53bd7ebb417ee9bac7db1cf9e0e8808 Luigi Bai 2020-09-09 17:08:59
Don't put status information into console.log. 19dab8ff60a0256f9511391d4088c1b8f88ac3db Luigi Bai 2020-09-09 17:06:24
Add more info to README. 9708b396b887bab284e0ad2c2d996ea2db0ff4d4 Luigi Bai 2020-09-09 12:37:24
Add info to README. 79672fad423a570d7eb05c4891dfabb511db5ab3 Luigi Bai 2020-09-09 00:20:23
Fixed searchNames. Now you can load cases where you know one of the parties. 0ad812704e27341a481a6de18c88a4f38bb61700 Luigi Bai 2020-09-08 23:43:48
Apparently updating addresses works, as long as you don't ask for more than 10 per minute. That seems to be working so far. 9276b2acb2cc3067c53fc16317a8960db3f6fcca Luigi Bai 2020-09-08 22:44:24
More information about how to use the software. Not much, yet. Enough to get started, I think. 393fce58bbcfeaee947cb2b0e17d81d16da49612 Luigi Bai 2020-09-08 22:31:02
Don't write to console.err if there's a rejectionFunction. 630b8ec76a21733a6bb93d98ca54563aadd61da4 Luigi Bai 2020-09-08 22:30:29
Remove trailing comma on all rows. e7605fa152a841eefc7fcab6bb7cfd64daf28b17 Luigi Bai 2020-09-07 22:43:36
Allow passing in query for Case.sqlAllCases() 346862c0720ef9054426048cc95caa8f6b068868 Luigi Bai 2020-09-07 22:43:09
This version works in MS-SQL. 4e3f33f464173c91c3f4ac4b6acb0235ed2d6720 Luigi Bai 2020-09-07 19:11:54
Add headers. 4da1ccbdb2e3e852050de3460bc9ac9ba061ed7a Luigi Bai 2020-09-07 16:28:52
loadDockets now works to local. 96ba8f7419f63be6612fe1efd3e24d5d30eaf20a Luigi Bai 2020-09-06 20:59:06
Fix for new column names. 5791d77e19585c494e51c6062ab45440e5af0903 Luigi Bai 2020-09-06 20:44:50
Add headers 32cb7264c06b28dc0404b715f24fd2d82b66633b Luigi Bai 2020-09-06 20:44:29
Have a working version of xfer - now transfers data to MS SQL Server from SQLITE3. 428e493bcc5b7f718a741db046d08905f47d0f1c Luigi Bai 2020-09-05 22:11:55
Stupid MS SQL Server assumes case insensitive collation for text/char columns. Who does that? 2700846d8c1233d27664fb74b582293ddb8cb32a Luigi Bai 2020-09-05 21:11:17
Commit 3036e8ca177024179b3a6e391876121e526cd213 - Clean up code.
Author: Luigi Bai
Author date (UTC): 2020-09-16 02:36
Committer name: Luigi Bai
Committer date (UTC): 2020-09-16 02:36
Parent(s): 0da4130e3e3a3e2e855f2ef28c4194fb6b5dfe10
Signing key:
Tree: 5b99a63287fa0dd9480935caca4a1fc198bfffa2
File Lines added Lines deleted
lib/HTTPConversation.class.js 8 8
lib/OdysseyInfo.class.js 17 13
updateAddress.js 2 2
File lib/HTTPConversation.class.js changed (mode: 100644) (index 88abdbd..63bc9db)
... ... module.exports = class {
51 51 this.debug && console.log(reqURL); this.debug && console.log(reqURL);
52 52
53 53 // With the data all set up, now connect and parse: // With the data all set up, now connect and parse:
54 let req = https.request(opts, (res) => {
55 res.headers["set-cookie"] && res.headers["set-cookie"].map((cookie) => { this.jar.setCookieSync(cookie, reqURL); });
54 let req = https.request(opts, res => {
55 res.headers["set-cookie"] && res.headers["set-cookie"].map(cookie => { this.jar.setCookieSync(cookie, reqURL); });
56 56
57 57 // What to do with incoming data: // What to do with incoming data:
58 58 let doc = ""; let doc = "";
59 res.on("data", (d) => {
59 res.on("data", d => {
60 60 this.verbose && console.log("postURLData", d.toString()); this.verbose && console.log("postURLData", d.toString());
61 61 if (args.parser) { if (args.parser) {
62 62 args.parser.parseChunk(d); args.parser.parseChunk(d);
 
... ... module.exports = class {
71 71
72 72 }); });
73 73
74 req.on('error', (e) => {
74 req.on('error', e => {
75 75 if (this.rejFN && "function" === typeof this.rejFN) if (this.rejFN && "function" === typeof this.rejFN)
76 76 this.rejFN({ fn: "HC.postURLData", url: reqURL, err: e }); this.rejFN({ fn: "HC.postURLData", url: reqURL, err: e });
77 77 else else
 
... ... module.exports = class {
106 106 this.debug && console.log(reqURL); this.debug && console.log(reqURL);
107 107
108 108 // With the data all set up, now connect and parse: // With the data all set up, now connect and parse:
109 let req = https.request(opts, (res) => {
110 res.headers["set-cookie"] && res.headers["set-cookie"].map((cookie) => { this.jar.setCookieSync(cookie, reqURL); });
109 let req = https.request(opts, res => {
110 res.headers["set-cookie"] && res.headers["set-cookie"].map(cookie => { this.jar.setCookieSync(cookie, reqURL); });
111 111
112 112 // What to do with incoming data: // What to do with incoming data:
113 113 let doc = ""; let doc = "";
114 res.on("data", (d) => {
114 res.on("data", d => {
115 115 this.verbose && console.log("getURL", d.toString()); this.verbose && console.log("getURL", d.toString());
116 116 if (args.parser) { if (args.parser) {
117 117 args.parser.parseChunk(d); args.parser.parseChunk(d);
 
... ... module.exports = class {
126 126
127 127 }); });
128 128
129 req.on('error', (e) => {
129 req.on('error', e => {
130 130 if (this.rejFN && "function" === typeof this.rejFN) if (this.rejFN && "function" === typeof this.rejFN)
131 131 this.rejFN({ fn: "HC.getURL", url: reqURL, err: e }); this.rejFN({ fn: "HC.getURL", url: reqURL, err: e });
132 132 else else
File lib/OdysseyInfo.class.js changed (mode: 100644) (index fa6c951..7a27705)
... ... function getCaseIDParser(oiObj, callback, errFN) {
29 29 let handler = new DomHandler((err, dom) => { let handler = new DomHandler((err, dom) => {
30 30 if (err) { if (err) {
31 31 errFN && "function" === typeof errFN && errFN({ msg: "caseIDParser", err: err }); errFN && "function" === typeof errFN && errFN({ msg: "caseIDParser", err: err });
32 return;
32 33 } else { } else {
33 34 // Find the containing node: // Find the containing node:
34 let idLinks = du.filter((node) => {
35 let idLinks = du.filter(node => {
35 36 return ("a" === node.name && "caseLink" === node.attribs.class); return ("a" === node.name && "caseLink" === node.attribs.class);
36 37 }, dom); }, dom);
37 38
38 39 if (1 !== idLinks.length) { if (1 !== idLinks.length) {
39 40 errFN && "function" === typeof errFN && errFN({ msg: "caseIDParser: unable to find caseID", html: du.getOuterHTML(dom) }); errFN && "function" === typeof errFN && errFN({ msg: "caseIDParser: unable to find caseID", html: du.getOuterHTML(dom) });
41 return;
40 42 } }
41 43
42 44 try { try {
 
... ... function getCaseInfoParser(oiObj, callback, errFN) {
63 65 let handler = new DomHandler((err, dom) => { let handler = new DomHandler((err, dom) => {
64 66 if (err) { if (err) {
65 67 errFN && "function" === typeof errFN && errFN({ msg: "caseInfoParser", err: err }); errFN && "function" === typeof errFN && errFN({ msg: "caseInfoParser", err: err });
68 return;
66 69 } else { } else {
67 70 oiObj.debug && console.log("Finding party info"); oiObj.debug && console.log("Finding party info");
68 71
69 72 // Find the containing node: // Find the containing node:
70 let partyInfo = du.filter((node) => {
73 let partyInfo = du.filter(node => {
71 74 return ("div" === node.name && "divPartyInformation_body" === node.attribs.id); return ("div" === node.name && "divPartyInformation_body" === node.attribs.id);
72 75 }, dom); }, dom);
73 76
74 77 if (1 !== partyInfo.length) { if (1 !== partyInfo.length) {
75 78 errFN && "function" === typeof errFN && errFN({ msg: "caseIDParser: unable to find party info", oi: oiObj }); errFN && "function" === typeof errFN && errFN({ msg: "caseIDParser: unable to find party info", oi: oiObj });
79 return;
76 80 } }
77 81
78 82 // Each party is in a <div class="col-md-8"/> // Each party is in a <div class="col-md-8"/>
79 let partyDivs = du.filter((node) => {
83 let partyDivs = du.filter(node => {
80 84 return ("div" === node.name && "col-md-8" === node.attribs.class); return ("div" === node.name && "col-md-8" === node.attribs.class);
81 85 }, partyInfo); }, partyInfo);
82 86
83 87 // Each of those has a <p><span>class</span>name</p> and <p>Address</p>: // Each of those has a <p><span>class</span>name</p> and <p>Address</p>:
84 88 oiObj.parties = []; oiObj.parties = [];
85 partyDivs.forEach((partyDiv) => {
89 partyDivs.forEach(partyDiv => {
86 90 let party = {}; let party = {};
87 let pDivs = du.filter((node) => {
91 let pDivs = du.filter(node => {
88 92 return ("p" === node.name); return ("p" === node.name);
89 93 }, partyDiv); }, partyDiv);
90 94
91 95 // pDivs[0]: <p><span>class</span>name</p> // pDivs[0]: <p><span>class</span>name</p>
92 du.filter((node) => {
96 du.filter(node => {
93 97 return ("span" === node.name); return ("span" === node.name);
94 }, pDivs[0]).forEach((span) => {
98 }, pDivs[0]).forEach(span => {
95 99 // SHOULD only be ONE of these: // SHOULD only be ONE of these:
96 party.role = du.filter((node) => {
100 party.role = du.filter(node => {
97 101 return ("text" === node.type); return ("text" === node.type);
98 }, span).map((val) => {
102 }, span).map(val => {
99 103 return val.data; return val.data;
100 104 }).join(" ").trim(); }).join(" ").trim();
101 105 // Get rid of the span for the next manipulation: // Get rid of the span for the next manipulation:
102 106 du.removeElement(span); du.removeElement(span);
103 107 }); });
104 party.name = du.filter((node) => {
108 party.name = du.filter(node => {
105 109 return ("text" === node.type); return ("text" === node.type);
106 }, pDivs[0]).map((val) => {
110 }, pDivs[0]).map(val => {
107 111 return val.data; return val.data;
108 112 }).join(" ").trim(); }).join(" ").trim();
109 113
110 114 // pDivs[1]: Address // pDivs[1]: Address
111 115 if (pDivs[1]) { if (pDivs[1]) {
112 116 try { try {
113 party.address = du.filter((node) => {
117 party.address = du.filter(node => {
114 118 return ("text" === node.type); return ("text" === node.type);
115 }, pDivs[1]).map((val) => {
119 }, pDivs[1]).map(val => {
116 120 return val.data; return val.data;
117 121 }); });
118 122 } catch (e) { } catch (e) {
File updateAddress.js changed (mode: 100644) (index 5c47ea9..8e0bfad)
... ... const Case = require("./lib/Case.class");
21 21 const OdysseyInfo = require("./lib/OdysseyInfo.class"); const OdysseyInfo = require("./lib/OdysseyInfo.class");
22 22
23 23 let opts = require("./creds")["SQLITE3"]; let opts = require("./creds")["SQLITE3"];
24 opts.connectCallback = (database) => {
24 opts.connectCallback = database => {
25 25 // Prep the databse to update each case as we get its updated info: // Prep the databse to update each case as we get its updated info:
26 26 let pDao = new DAO.PartyCtlr(database); let pDao = new DAO.PartyCtlr(database);
27 27 pDao.stmts.update = pDao.prepare( pDao.stmts.update = pDao.prepare(
 
... ... opts.connectCallback = (database) => {
76 76 Promise.all(promises) Promise.all(promises)
77 77 .finally(() => { .finally(() => {
78 78 // Shut down the database, and the HTTPS agent: // Shut down the database, and the HTTPS agent:
79 require("./lib/httpOptions").getGlobalAgent().destroy();
80 79 pDao.finalize(); pDao.finalize();
81 80 cDao.finalize(); cDao.finalize();
82 81 DAO.disconnect(database); DAO.disconnect(database);
82 require("./lib/httpOptions").getGlobalAgent().destroy();
83 83 }) })
84 84 ; ;
85 85 } }
Hints:
Before first commit, do not forget to setup your git environment:
git config --global user.name "your_name_here"
git config --global user.email "your@email_here"

Clone this repository using HTTP(S):
git clone https://rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using ssh (do not forget to upload a key first):
git clone ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using git:
git clone git://git.rocketgit.com/user/datajams-lbai/datajams-evictions

You are allowed to anonymously push to this repository.
This means that your pushed commits will automatically be transformed into a merge request:
... clone the repository ...
... make some changes and some commits ...
git push origin main