datajams-lbai / datajams-evictions (public) (License: AGPLv3) (since 2020-09-01) (hash sha1)
Extracting Harris County evictions information for Houston DataJams
Clone URLs: https://rocketgit.com/user/datajams-lbai/datajams-evictions ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions git://git.rocketgit.com/user/datajams-lbai/datajams-evictions
first
List of commits:
Subject Hash Author Date (UTC)
loadDockets now works to local. 96ba8f7419f63be6612fe1efd3e24d5d30eaf20a Luigi Bai 2020-09-06 20:59:06
Fix for new column names. 5791d77e19585c494e51c6062ab45440e5af0903 Luigi Bai 2020-09-06 20:44:50
Add headers 32cb7264c06b28dc0404b715f24fd2d82b66633b Luigi Bai 2020-09-06 20:44:29
Have a working version of xfer - now transfers data to MS SQL Server from SQLITE3. 428e493bcc5b7f718a741db046d08905f47d0f1c Luigi Bai 2020-09-05 22:11:55
Stupid MS SQL Server assumes case insensitive collation for text/char columns. Who does that? 2700846d8c1233d27664fb74b582293ddb8cb32a Luigi Bai 2020-09-05 21:11:17
Fixed MS schema - 4096 too big for varchar. ca3870f7e244d83e58d74c7f231baacbc19ba8fa Luigi Bai 2020-09-04 17:04:05
Remove interstitial console.log 19481f6692d9f93e75aa173bee9716261ef54db0 Luigi Bai 2020-09-04 12:07:33
Added license/copyright info. Added shim for Promise. Rewrote xferData with await and shim. 3930998d32fb2a19239e63b1b07bf1804c7ea46e Luigi Bai 2020-09-04 12:04:32
Fixed a typo. Ugh. 64718362e042d4a987d3161f999de087edb7bc8a Luigi Bai 2020-09-04 03:59:37
Must have an example of the creds.js file to learn from. 13d0e75c21bf8194c53e148e37cc6f85d8152de3 Luigi Bai 2020-09-04 03:59:16
Transfer local data to cloud based MS SQL-Server database. 8035b6dac5ce05ced62474409ab46ca490a6db4a Luigi Bai 2020-09-04 03:50:32
Learned that package-lock.json should also be committed. 20222232bd2e5388b76d3330e6f695b5aacfe393 Luigi Bai 2020-09-03 11:04:05
Fixed SPDX license expression. c894ba842750ff63035c850095ac93399495f4bc Luigi Bai 2020-09-03 11:01:27
Updated README. Added .gitignore, node/npm package.json, and the schema definitions for the data stores. a5baa30d1360514e217250737abf3a4518ab6a65 Luigi Bai 2020-09-02 17:35:09
Initial commit f67ef5b180d66c95e84ee5c5aa0a893376afa5c4 datajams-lbai 2020-09-02 02:28:52
Commit 96ba8f7419f63be6612fe1efd3e24d5d30eaf20a - loadDockets now works to local.
Author: Luigi Bai
Author date (UTC): 2020-09-06 20:59
Committer name: Luigi Bai
Committer date (UTC): 2020-09-07 16:22
Parent(s): 5791d77e19585c494e51c6062ab45440e5af0903
Signing key:
Tree: f37eb90fa6e8fa262ebddfed51fe2d00be5b8e7e
File Lines added Lines deleted
lib/Case.class.js 388 0
lib/Court.class.js 171 0
lib/Docket.class.js 136 0
lib/HTTPConversation.class.js 133 0
lib/OdysseyInfo.class.js 282 0
lib/httpOptions.js 21 20
lib/sqlDAO.js 0 1
loadDockets.js 111 0
schema.mssql.sql 1 1
searchNames.js 32 0
sqdump 12 0
updateStatus.js 67 0
File lib/Case.class.js added (mode: 100644) (index 0000000..6f1b438)
1 /*
2 This file is part of datajams-evictions.
3
4 datajams-evictions is free software: you can redistribute it and/or modify
5 it under the terms of the GNU Affero General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 datajams-evictions is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU Affero General Public License for more details.
13
14 You should have received a copy of the GNU Affero General Public License
15 along with datajams-evictions. If not, see <https://www.gnu.org/licenses/>.
16
17 Copyright 2020 Luigi Bai
18 */
19 const https = require("https");
20 const httpOpts = require("./httpOptions");
21 const HTTPConversation = require("./HTTPConversation.class");
22 const util = require("util");
23 const hp = require("htmlparser2");
24 const DomHandler = require("domhandler").DomHandler;
25 const du = require("domutils");
26 const DAO = require("./sqlDAO");
27 const OdysseyInfo = require("./OdysseyInfo.class");
28
29 // ---------------------------------------------------
30 // Some helper functions:
31 function checkPreviousSiblings(node, test) {
32 let ret = false;
33
34 if (null !== node) {
35 if (null !== node.prev) {
36 let prev = node.prev;
37 ret = test(prev) || checkPreviousSiblings(prev, test);
38 }
39 }
40
41 return ret;
42 }
43
44 function findAndSet(label, member, nodeSet, obj) {
45 // Should work for any label and member:
46 let cnNode = du.filter((node) => {
47 return checkPreviousSiblings(node, (testNode) => {
48 if ("span" === testNode.name && "label" === testNode.attribs.class) {
49 return ("text" === testNode.children[0].type && label === testNode.children[0].data);
50 } else {
51 return false;
52 }
53 });
54 }, nodeSet);
55
56 if (0 === cnNode.length) {
57 // ERROR
58 console.error("Warning, no value found: ", label, member, obj);
59 } else {
60 if (0 === cnNode[0].children.length) {
61 // ERROR
62 console.error("Warning, no value found: ", label, member, obj);
63 } else {
64 obj[member] = cnNode[0].children[0].data.trim();
65 }
66 }
67 }
68
69 function getCaseInfo(nodes, obj) {
70 let ci = du.filter((node) => {
71 return ("div" === node.name && "caseInfo" === node.attribs.id);
72 }, nodes);
73
74 // get all the labels:
75 let labels = du.filter((node) => {
76 return ("span" === node.name && "value" === node.attribs.class);
77 }, ci);
78
79 findAndSet("Case Number:", "caseNumber", labels, obj);
80 findAndSet("Filed Date:", "filedDate", labels, obj);
81 findAndSet("Case Status:", "caseStatus", labels, obj);
82 };
83
84 function getCivilInfo(nodes, obj) {
85 let ci = du.filter((node) => {
86 return ("div" === node.name && "civilInfo" === node.attribs.id);
87 }, nodes);
88
89 if (0 === ci.length) {
90 // Odd. This should be a civil claim, and yet there's no civil info.
91 // May have found a criminal offense?
92 } else {
93
94 // get all the labels:
95 let labels = du.filter((node) => {
96 return ("span" === node.name && "value" === node.attribs.class);
97 }, ci);
98
99 // findAndSet("Nature of Claim:", "natureOfClaim", labels, obj);
100 findAndSet("Claim Amount:", "claimAmount", labels, obj);
101 }
102 };
103
104 function getHearingInfo(nodes, obj) {
105 let ci = du.filter((node) => {
106 return ("div" === node.name && "eventInfo" === node.attribs.id);
107 }, nodes);
108
109 // get all the hearings:
110 let hearings = du.filter((node) => {
111 return ("div" === node.name && ("odd" === node.attribs.class || "even" === node.attribs.class));
112 }, ci);
113
114 let hearingList = [];
115
116 hearings.forEach((hearingDiv) => {
117 let hearing = {};
118
119 // get all the labels:
120 let labels = du.filter((node) => {
121 return ("span" === node.name && "value" === node.attribs.class);
122 }, hearingDiv);
123
124 findAndSet("Hearing Description:", "hearingDescription", labels, hearing);
125 findAndSet("Hearing Date/Time:", "hearingDateTime", labels, hearing);
126 findAndSet("Hearing Result/Cancellation:", "hearingResult", labels, hearing);
127 findAndSet("Hearing Result/Cancellation Date:", "hearingResultDateTime", labels, hearing);
128
129 hearingList.push(hearing);
130 });
131
132 obj.hearings = hearingList;
133 };
134
135 function getPartyInfo(nodes, obj) {
136 let ci = du.filter((node) => {
137 return ("div" === node.name && "partyInfo" === node.attribs.id);
138 }, nodes);
139
140 // get all the parties:
141 let parties = du.filter((node) => {
142 return ("div" === node.name && ("odd" === node.attribs.class || "even" === node.attribs.class));
143 }, ci);
144
145 let partyList = [];
146
147 // For each party:
148 parties.forEach((partyDiv) => {
149 let party = {};
150
151 // get all the labels:
152 let labels = du.filter((node) => {
153 return ("span" === node.name && "value" === node.attribs.class);
154 }, partyDiv);
155
156 findAndSet("Party Name:", "name", labels, party);
157 findAndSet("Party Type:", "role", labels, party);
158
159 partyList.push(party);
160 });
161
162 obj.parties = partyList;
163 };
164
165 function getEventInfo(nodes, obj) {
166 let ci = du.filter((node) => {
167 return ("div" === node.name && "filingInfo" === node.attribs.id);
168 }, nodes);
169
170 // get all the parties:
171 let events = du.filter((node) => {
172 return ("div" === node.name && ("odd" === node.attribs.class || "even" === node.attribs.class));
173 }, ci);
174
175 let eventList = [];
176
177 // For each party:
178 events.forEach((partyDiv) => {
179 let party = {};
180
181 // get all the labels:
182 let labels = du.filter((node) => {
183 return ("span" === node.name && "value" === node.attribs.class);
184 }, partyDiv);
185
186 findAndSet("Event Description:", "description", labels, party);
187 findAndSet("Date Added:", "dateAdded", labels, party);
188
189 eventList.push(party);
190 });
191
192 obj.events = eventList;
193 };
194 // ---------------------------------------------------
195
196 // Expose just the class:
197 class Case {
198 constructor(docket, url) {
199 this.docket = docket;
200
201 if (!url) {
202 throw new Error("Must include a URL.");
203 }
204 this.URL = url;
205 };
206
207 static sqlAllCases(database, rowCB, completeCB) {
208 database.each(
209 "SELECT case_URL from cases",
210 (err, row) => {
211 if (err) {
212 console.error("Case.sqlAllCases", err);
213 } else {
214 rowCB && "function" === typeof rowCB && rowCB(new Case(null, row["case_URL"]));
215 }
216 },
217 completeCB
218 );
219 };
220
221 static findCasesByBusiness(name, cb) {
222 let hc = new HTTPConversation({
223 debug: false
224 });
225
226 let host = "jpwebsite.harriscountytx.gov";
227
228 let handler = new DomHandler((err, dom) => {
229 if (err) {
230 console.error("Cases.findCasesByBusiness", err);
231 } else {
232 if (cb && "function" === typeof cb) {
233 // Find the containing node:
234 let caseTable = du.filter((node) => {
235 return ("table" === node.name && "cases" === node.attribs.id);
236 }, dom);
237
238 // Now get the table rows:
239 let caseRows = du.filter((node) => {
240 return ("tr" === node.name);
241 }, caseTable);
242
243 // For each of the rows, grab the case URL:
244 let caseHrefs = du.filter((node) => {
245 return ("a" === node.name && "td" === node.parent.name && 'text' === node.parent.prev.type && null === node.parent.prev.prev);
246 }, caseRows);
247
248 let cases = caseHrefs.map((aNode) => { return "https://"+host+aNode.attribs.href; });
249 cb(cases);
250 }
251 };
252 }, {
253 normalizeWhitespace: true
254 });
255
256 // First, get "https://jpwebsite.harriscountytx.gov/FindMyCase/search.jsp"
257 hc.getURL({
258 reqPath: "/FindMyCase/search.jsp",
259 reqHost: host,
260 callback: (doc) => {
261 // Next, post the search:
262 hc.postURLData({
263 reqPath: "/FindMyCase/SearchForMyCases",
264 reqHost: host,
265 data: {
266 "case": "",
267 "dlNumber": "",
268 "lastName": "",
269 "firstName": "",
270 "dob": "",
271 "businessName": name,
272 "court": "00",
273 "criteria": "businessNameSearch"
274 },
275 parser: new hp.Parser(handler)
276 });
277 }
278 });
279
280 };
281
282 loadCaseFromURL() {
283 return new Promise((resFN, rejFN) => {
284 let opts = new httpOpts();
285
286 // With the data all set up, now connect and parse:
287 let req = https.request(this.URL, opts, (res) => {
288 let handler = new DomHandler((err, dom) => {
289 if (err) {
290 rejFN({ url: this.URL, err: err });
291 } else {
292 // Find the containing node:
293 let dataDiv = du.filter((node) => {
294 return ("div" === node.name && "contentWithNavBar" === node.attribs.class);
295 }, dom);
296
297 // Get the civil info:
298 getCaseInfo(dataDiv, this);
299 getCivilInfo(dataDiv, this);
300 getPartyInfo(dataDiv, this);
301 getHearingInfo(dataDiv, this);
302 getEventInfo(dataDiv, this);
303
304 resFN && "function" === typeof resFN && resFN(this);
305 };
306 }, {
307 normalizeWhitespace: true
308 });
309 let parser = new hp.Parser(handler);
310
311 // What to do with incoming data:
312 res.on("data", (d) => {
313 parser.parseChunk(d);
314 }).on("end", () => {
315 parser.done();
316 });
317
318 });
319
320 req.on('error', e => {
321 rejFN({ msg: "Load Case from URL HTTP", err: e });
322 });
323
324 // Ok GO:
325 req.end();
326 });
327 };
328
329 storeSQL(database) {
330 let caseObj = this;
331
332 if (caseObj.docket) {
333 let ctlDocket = new DAO.DocketCtlr(database);
334 ctlDocket.insert({
335 $pct: caseObj.docket.court.precinct,
336 $plc: caseObj.docket.court.place,
337 $dt: caseObj.docket.date.substr(6)+"-"+
338 caseObj.docket.date.substr(0, 2)+"-"+
339 caseObj.docket.date.substr(3, 2)+" "+
340 caseObj.docket.time,
341 $url: caseObj.docket.URL.href
342 });
343 ctlDocket.finalize();
344
345 let ctlDC = new DAO.DocketCaseCtlr(database);
346 ctlDC.insert({
347 $pct: caseObj.docket.court.precinct,
348 $plc: caseObj.docket.court.place,
349 $dt: caseObj.docket.date.substr(6)+"-"+
350 caseObj.docket.date.substr(0, 2)+"-"+
351 caseObj.docket.date.substr(3, 2)+" "+
352 caseObj.docket.time,
353 $cnum: caseObj.caseNumber,
354 $claim: caseObj.claimAmount
355 });
356 ctlDC.finalize();
357 }
358
359 let ctlCase = new DAO.CaseCtlr(database);
360 ctlCase.insert({
361 $cnum: caseObj.caseNumber,
362 $curl: caseObj.URL
363 });
364 ctlCase.finalize();
365
366 let ctlParty = new DAO.PartyCtlr(database);
367 caseObj.parties.forEach((party) => {
368 ctlParty.insert({
369 $cnum: caseObj.caseNumber,
370 $role: party.role,
371 $name: party.name
372 });
373 });
374 ctlParty.finalize();
375
376 let ctlEvent = new DAO.EventCtlr(database);
377 caseObj.events.forEach((event) => {
378 ctlEvent.insert({
379 $cnum: caseObj.caseNumber,
380 $desc: event.description,
381 $date: event.dateAdded
382 });
383 });
384 ctlEvent.finalize();
385 };
386 };
387
388 module.exports = Case;
File lib/Court.class.js added (mode: 100644) (index 0000000..743336b)
1 /*
2 This file is part of datajams-evictions.
3
4 datajams-evictions is free software: you can redistribute it and/or modify
5 it under the terms of the GNU Affero General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 datajams-evictions is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU Affero General Public License for more details.
13
14 You should have received a copy of the GNU Affero General Public License
15 along with datajams-evictions. If not, see <https://www.gnu.org/licenses/>.
16
17 Copyright 2020 Luigi Bai
18 */
19 const https = require("https");
20 const httpOpts = require("./httpOptions");
21 const util = require("util");
22 const hp = require("htmlparser2");
23 const DomHandler = require("domhandler").DomHandler;
24 const du = require("domutils");
25 const Docket = require("./Docket.class");
26
27 module.exports = class Court {
28 constructor(argv) {
29 if (! argv) {
30 throw new Error("Must have arguments.");
31 }
32
33 if (! argv.precinct) {
34 throw new Error("Must have a precinct.");
35 }
36 this.precinct = argv.precinct;
37
38 if (! argv.place) {
39 throw new Error("Must have a place.");
40 }
41 this.place = argv.place;
42 };
43
44 toString() {
45 return JSON.stringify({ precinct: this.precinct, place: this.place });
46 };
47
48 getDockets(argv) {
49 if (! argv) {
50 throw new Error("Must have arguments.");
51 }
52
53 return new Promise((resFN, rejFN) => {
54 let startDate = argv.startDate || new Date();
55
56 let endDate = argv.endDate;
57 if (! argv.endDate) {
58 this.endDate = new Date(this.startDate.valueOf());
59 this.endDate.setDate(this.startDate.getDate() + 6);
60 }
61
62 let docketType = argv.docketType || "Eviction Docket";
63
64 let params = new URLSearchParams({
65 "court": (this.precinct * 10 + this.place),
66 "fdate": (startDate.getMonth() + 1)+"/"+(startDate.getDate())+"/"+(startDate.getFullYear()),
67 "tdate": (endDate.getMonth() + 1)+"/"+(endDate.getDate())+"/"+(endDate.getFullYear())
68 });
69 let opts = new httpOpts();
70 let reqURL = "https://jpwebsite.harriscountytx.gov/WebDockets/GetDocketSummary?"+params.toString();
71
72 this.dockets = [];
73
74 // With the data all set up, now connect and parse:
75 let req = https.request(reqURL, opts, (res) => {
76 let handler = new DomHandler((err, dom) => {
77 if (err) {
78 rejFN({ url: reqURL, err: err });
79 } else {
80 let isEventDescription = (node, type) => {
81 let ret = ("div" === node.name && "record" === node.attribs.class);
82 if (ret && undefined !== type) {
83 ret = false;
84 // Dig a little deeper. node.children[x].name="span" && node.children[x].attribs.class="eventDescription"
85 for (let i = 0; i < node.children.length; i++) {
86 let ci = node.children[i];
87 if ("span" === ci.name && "eventDescription" === ci.attribs.class) {
88 // One of the children has to be "text" and match "type":
89 for (let j = 0; !ret && j < ci.children.length; j++) {
90 let cj = ci.children[j];
91 ret = ("text" === cj.type && type === cj.data);
92 }
93 }
94 }
95 }
96 return ret;
97 };
98
99 // Find the containing node:
100 let events = du.filter((node) => {
101 return ("div" === node.name && "contentWithNavBar" === node.attribs.class);
102 }, dom)[0];
103
104 // Ok. Need to walk the DOM object with the lousy tools I have in javascript.
105 let docketPromises = [];
106 for (let di = 0; di < events.children.length; di++) {
107 // Looking for: node.name = "div", node.attribs.class = "record"
108 let node = events.children[di];
109 if (isEventDescription(node, docketType)) {
110 // Work backwards to find the div.docketTimeHeader:
111 let found = false;
112 for (let dj = di; !found && dj >= 0; dj--) {
113 let psib = events.children[dj];
114 found = ("div" === psib.name && "docketTimeHeader" === psib.attribs.class);
115 if (found) {
116 // Dig out the href:
117 for (let si = 0; si < psib.children.length; si++) {
118 let snode = psib.children[si];
119 if ("a" === snode.name) {
120 let hURL = new URL(snode.attribs.href, reqURL);
121 let docket = new Docket({
122 court: this,
123 URL: hURL,
124 type: docketType
125 });
126 this.dockets.push(docket);
127 docketPromises.push(docket.getCases(argv.caseCallback));
128 }
129 }
130 }
131 }
132 }
133 }
134 // At this point, not much to return. Let the embedded
135 // promises do their work?
136 Promise.all(docketPromises)
137 .then(val => {
138 resFN({
139 court: this,
140 dockets: val.map(v => {
141 if ((v && v instanceof Array && v.length > 0) || v)
142 return v;
143 })
144 });
145 })
146 .catch(err => { rejFN(err); })
147 ;
148 };
149 }, {
150 normalizeWhitespace: true
151 });
152 let parser = new hp.Parser(handler);
153
154 // What to do with incoming data:
155 res.on("data", (d) => {
156 parser.parseChunk(d);
157 }).on("end", () => {
158 parser.done();
159 });
160
161 });
162
163 req.on('error', (e) => {
164 console.error(reqURL, e);
165 });
166
167 // Ok GO:
168 req.end();
169 });
170 }
171 };
File lib/Docket.class.js added (mode: 100644) (index 0000000..90b150d)
1 /*
2 This file is part of datajams-evictions.
3
4 datajams-evictions is free software: you can redistribute it and/or modify
5 it under the terms of the GNU Affero General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 datajams-evictions is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU Affero General Public License for more details.
13
14 You should have received a copy of the GNU Affero General Public License
15 along with datajams-evictions. If not, see <https://www.gnu.org/licenses/>.
16
17 Copyright 2020 Luigi Bai
18 */
19 const https = require("https");
20 const httpOpts = require("./httpOptions");
21 const util = require("util");
22 const hp = require("htmlparser2");
23 const DomHandler = require("domhandler").DomHandler;
24 const du = require("domutils");
25 const Case = require("./Case.class");
26
27 module.exports = class Docket {
28 constructor(argv) {
29 if (!argv.court) {
30 throw new Error("Must have a parent court.");
31 }
32 this.court = argv.court;
33
34 this.docketType = argv.docketType || "Eviction Docket";
35
36 if (!argv.URL) {
37 throw new Error("Must include a URL.");
38 }
39 this.URL = argv.URL;
40 this.date = this.URL.searchParams.get("date");
41 let oTime = this.URL.searchParams.get("time");
42 this.time = oTime.substr(0, 2) + ":" + oTime.substr(2, 2) + ":" + oTime.substr(4);
43 };
44
45 toString() {
46 return JSON.stringify({
47 court: this.court.toString(),
48 docketType: this.docketType,
49 url: this.url.toString(),
50 cases: this.cases
51 });
52 };
53
54 getCases(caseCallback) {
55 return new Promise((resFN, rejFN) => {
56 let opts = new httpOpts();
57
58 // With the data all set up, now connect and parse:
59 let req = https.request(this.URL, opts, (res) => {
60 let handler = new DomHandler((err, dom) => {
61 if (err) {
62 rejFN({ url: this.URL, err: err });
63 } else {
64 // Find the containing node:
65 let events = du.filter((node) => {
66 return ("div" === node.name && "contentWithNavBar" === node.attribs.class);
67 }, dom);
68 // Then get the div.caseDetails nodes within them:
69 let cases = du.filter((node) => {
70 return ("div" === node.name && "caseDetails" === node.attribs.class);
71 }, events);
72 // From those, get the span.emphasis nodes:
73 let urlSpans = du.filter((node) => {
74 if ("span" === node.name && "emphasis" === node.attribs.class) {
75 // Check the siblings to make sure they're the event type we want:
76 let sibList = du.filter((sib) => {
77 return ("text" === sib.type && sib.data.trim() === this.docketType);
78 }, du.getSiblings(node));
79 return (sibList.length > 0);
80 } else {
81 return false;
82 }
83 }, cases);
84
85 // In those, grab the "a" nodes:
86 let urlNodes = du.filter((node) => {
87 return ("a" === node.name);
88 }, urlSpans);
89
90 // For each of these, grab the hrefs:
91 this.cases = urlNodes.map((node) => {
92 return new Case(this, new URL(node.attribs.href, this.URL).toString());
93 });
94
95 Promise.all(
96 this.cases.map((caseObj) => {
97 return caseObj.loadCaseFromURL()
98 .then(caseO => {
99 caseCallback && "function" === typeof caseCallback && caseCallback(caseO);
100 })
101 .catch(err => { rejFN(err); })
102 ;
103 })
104 )
105 .then(val => {
106 resFN({
107 docket: this,
108 cases: val.map(v => { return v; })
109 });
110 })
111 .catch(err => { rejFN(err); })
112 ;
113 };
114 }, {
115 normalizeWhitespace: true
116 });
117 let parser = new hp.Parser(handler);
118
119 // What to do with incoming data:
120 res.on("data", (d) => {
121 parser.parseChunk(d);
122 }).on("end", () => {
123 parser.done();
124 });
125
126 });
127
128 req.on('error', (e) => {
129 rejFN({ msg: "Load Docket HTTP", err: e });
130 });
131
132 // Ok GO:
133 req.end();
134 });
135 };
136 };
File lib/HTTPConversation.class.js added (mode: 100644) (index 0000000..5c32b16)
1 /*
2 This file is part of datajams-evictions.
3
4 datajams-evictions is free software: you can redistribute it and/or modify
5 it under the terms of the GNU Affero General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 datajams-evictions is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU Affero General Public License for more details.
13
14 You should have received a copy of the GNU Affero General Public License
15 along with datajams-evictions. If not, see <https://www.gnu.org/licenses/>.
16
17 Copyright 2020 Luigi Bai
18 */
19 const https = require("https");
20 const httpOpts = require("./httpOptions");
21 const tough = require("tough-cookie");
22
23 module.exports = class {
24 constructor(argv) {
25 let opts = argv || {};
26 this.debug = opts.debug;
27 this.verbose = opts.verbose;
28 this.jar = (new tough.CookieJar());
29 };
30
31 postURLData(argv) {
32 let args = argv || {};
33
34 if (! args.reqPath) throw "Must have a request path.";
35 if (! args.reqHost) throw "Must have a request host.";
36
37 let opts = new httpOpts();
38 opts.host = args.reqHost;
39 opts.path = args.reqPath;
40 opts.method = "POST";
41 opts.headers["content-type"] = "application/x-www-form-urlencoded";
42 delete opts.headers["Accept-Encoding"];
43 this.referURL && (opts.headers["referer"] = this.referURL);
44
45 let reqURL = opts.protocol+"//"+opts.host+opts.path;
46 opts.headers["cookie"] = this.jar.getSetCookieStringsSync(reqURL);
47
48 this.debug && console.log(opts);
49 this.debug && console.log(reqURL);
50
51 // With the data all set up, now connect and parse:
52 let req = https.request(opts, (res) => {
53 this.debug && console.log(res);
54 res.headers["set-cookie"] && res.headers["set-cookie"].map((cookie) => { this.jar.setCookieSync(cookie, reqURL); });
55
56 // What to do with incoming data:
57 let doc = "";
58 res.on("data", (d) => {
59 this.verbose && console.log("postURLData", d.toString());
60 if (args.parser) {
61 args.parser.parseChunk(d);
62 } else {
63 doc = doc + d;
64 }
65 }).on("end", () => {
66 this.referURL = reqURL;
67 args.parser && args.parser.done();
68 args.callback && "function" === typeof args.callback && args.callback(doc);
69 });
70
71 });
72
73 req.on('error', (e) => {
74 console.error(reqURL, e);
75 });
76
77 let us = new URLSearchParams(args.data);
78 this.debug && console.log("data: ", us.toString());
79 req.write(us.toString());
80
81 // Ok GO:
82 req.end();
83 };
84
85 getURL(argv) {
86 let args = argv || {};
87
88 if (! args.reqPath) throw "Must have a request path.";
89 if (! args.reqHost) throw "Must have a request host.";
90
91 let opts = new httpOpts();
92 opts.protocol = "https:";
93 opts.host = args.reqHost;
94 opts.path = args.reqPath;
95 opts.method = "GET";
96 this.referURL && (opts.headers["referer"] = this.referURL);
97
98 let reqURL = opts.protocol+"//"+opts.host+opts.path;
99 opts.headers["cookie"] = this.jar.getSetCookieStringsSync(reqURL);
100
101 this.debug && console.log(opts);
102 this.debug && console.log(reqURL);
103
104 // With the data all set up, now connect and parse:
105 let req = https.request(opts, (res) => {
106 this.debug && console.log(res);
107 res.headers["set-cookie"] && res.headers["set-cookie"].map((cookie) => { this.jar.setCookieSync(cookie, reqURL); });
108
109 // What to do with incoming data:
110 let doc = "";
111 res.on("data", (d) => {
112 this.verbose && console.log("getURL", d.toString());
113 if (args.parser) {
114 args.parser.parseChunk(d);
115 } else {
116 doc = doc + d;
117 }
118 }).on("end", () => {
119 this.referURL = reqURL;
120 args.parser && args.parser.done();
121 args.callback && "function" === typeof args.callback && args.callback(doc);
122 });
123
124 });
125
126 req.on('error', (e) => {
127 console.error(reqURL, e);
128 });
129
130 // Ok GO:
131 req.end();
132 };
133 };
File lib/OdysseyInfo.class.js added (mode: 100644) (index 0000000..e021696)
1 const https = require("https");
2 const httpOpts = require("./httpOptions");
3 const HTTPConversation = require("./HTTPConversation.class");
4 const util = require("util");
5 const hp = require("htmlparser2");
6 const DomHandler = require("domhandler").DomHandler;
7 const du = require("domutils");
8
9 /*
10 function postURLData(oiObj, reqPath, data, cb) {
11 let opts = new httpOpts();
12 opts.host = "odysseyportal.harriscountytx.gov";
13 opts.path = reqPath;
14 opts.method = "POST";
15 opts.headers["content-type"] = "application/x-www-form-urlencoded";
16 oiObj.referURL && (opts.headers["referer"] = oiObj.referURL);
17
18 let reqURL = opts.protocol+"//"+opts.host+reqPath;
19 opts.headers["cookie"] = oiObj.jar.getSetCookieStringsSync(reqURL);
20
21 oiObj.debug && console.log(opts);
22 oiObj.debug && console.log(reqURL);
23
24 // With the data all set up, now connect and parse:
25 let req = https.request(opts, (res) => {
26 oiObj.debug && console.log(res.headers);
27 res.headers["set-cookie"] && res.headers["set-cookie"].map((cookie) => { oiObj.jar.setCookieSync(cookie, reqURL); });
28
29 // What to do with incoming data:
30 let doc = "";
31 res.on("data", (d) => {
32 oiObj.verbose && console.log("postURLData", d.toString());
33 doc = doc + d;
34 }).on("end", () => {
35 oiObj.referURL = reqURL;
36 cb(doc);
37 });
38
39 });
40
41 req.on('error', (e) => {
42 console.error(reqURL, e);
43 });
44
45 let us = new URLSearchParams(data);
46 oiObj.debug && console.log("data: ", us.toString());
47 req.write(us.toString());
48
49 // Ok GO:
50 req.end();
51 };
52
53 function getURL(oiObj, argv) {
54 if (!argv.requestPath) {
55 throw new Error("Must include a request path.");
56 }
57
58 let opts = new httpOpts();
59 opts.protocol = "https:";
60 opts.host = "odysseyportal.harriscountytx.gov";
61 opts.path = argv.requestPath;
62 opts.method = "GET";
63 oiObj.referURL && (opts.headers["referer"] = oiObj.referURL);
64
65 let reqURL = opts.protocol+"//"+opts.host+"/"+argv.requestPath;
66 opts.headers["cookie"] = oiObj.jar.getSetCookieStringsSync(reqURL);
67
68 oiObj.debug && console.log(opts);
69 oiObj.debug && console.log(reqURL);
70
71 // With the data all set up, now connect and parse:
72 let req = https.request(opts, (res) => {
73 oiObj.debug && console.log(res.headers);
74 res.headers["set-cookie"] && res.headers["set-cookie"].map((cookie) => { oiObj.jar.setCookieSync(cookie, argv.requestURL); });
75
76 // What to do with incoming data:
77 let doc = "";
78 res.on("data", (d) => {
79 oiObj.verbose && console.log("getURL", d.toString());
80 if (argv.parser) {
81 argv.parser.parseChunk(d);
82 } else {
83 doc = doc + d;
84 }
85 }).on("end", () => {
86 oiObj.referURL = argv.requestURL;
87 argv.parser && argv.parser.done();
88 argv.callback && argv.callback(doc);
89 });
90
91 });
92
93 req.on('error', (e) => {
94 console.error(reqURL, e);
95 });
96
97 // Ok GO:
98 req.end();
99 };
100 */
101
102 function getCaseIDParser(oiObj, callback) {
103 // <a class="caseLink" data-caseid="jBTdKQrus6pMdAW2SPPMBA2" data-caseid-num="6402180">205100178940</a>
104 let handler = new DomHandler((err, dom) => {
105 if (err) {
106 console.error("caseIDParser", err);
107 } else {
108 // Find the containing node:
109 let idLinks = du.filter((node) => {
110 return ("a" === node.name && "caseLink" === node.attribs.class);
111 }, dom);
112
113 if (1 !== idLinks.length) {
114 throw new Error("Unable to find caseID.");
115 }
116
117 oiObj.caseID = idLinks[0].attribs["data-caseid"];
118
119 // Now use the callback to process the hrefs:
120 (callback && "function" === typeof callback) && callback();
121 };
122 }, {
123 normalizeWhitespace: true
124 });
125
126 return new hp.Parser(handler);
127 };
128
129 function getCaseInfoParser(oiObj, callback) {
130 // <a class="caseLink" data-caseid="jBTdKQrus6pMdAW2SPPMBA2" data-caseid-num="6402180">205100178940</a>
131 let handler = new DomHandler((err, dom) => {
132 if (err) {
133 console.error("caseIDParser", err);
134 } else {
135 // Find the containing node:
136 let partyInfo = du.filter((node) => {
137 return ("div" === node.name && "divPartyInformation_body" === node.attribs.id);
138 }, dom);
139
140 if (1 !== partyInfo.length) {
141 throw new Error("Unable to find party info.");
142 }
143
144 // Each party is in a <div class="col-md-8"/>
145 let partyDivs = du.filter((node) => {
146 return ("div" === node.name && "col-md-8" === node.attribs.class);
147 }, partyInfo);
148
149 // Each of those has a <p><span>class</span>name</p> and <p>Address</p>:
150 this.parties = [];
151 partyDivs.forEach((partyDiv) => {
152 let party = {};
153 let pDivs = du.filter((node) => {
154 return ("p" === node.name);
155 }, partyDiv);
156
157 // pDivs[0]: <p><span>class</span>name</p>
158 du.filter((node) => {
159 return ("span" === node.name);
160 }, pDivs[0]).forEach((span) => {
161 // SHOULD only be ONE of these:
162 party.role = du.filter((node) => {
163 return ("text" === node.type);
164 }, span).map((val) => {
165 return val.data;
166 }).join(" ").trim();
167 // Get rid of the span for the next manipulation:
168 du.removeElement(span);
169 });
170 party.name = du.filter((node) => {
171 return ("text" === node.type);
172 }, pDivs[0]).map((val) => {
173 return val.data;
174 }).join(" ").trim();
175
176 // pDivs[1]: Address
177 party.address = du.filter((node) => {
178 return ("text" === node.type);
179 }, pDivs[1]).map((val) => {
180 return val.data;
181 });
182
183 this.parties.push(party);
184 });
185
186 // Now do the callback:
187 (callback && "function" === typeof callback) && callback(oiObj.parties);
188 };
189 }, {
190 normalizeWhitespace: true
191 });
192
193 return new hp.Parser(handler);
194 };
195
196 module.exports = class OdysseyInfo {
197 constructor(caseNumber, argv) {
198 if (! caseNumber) {
199 throw new Error("Must define a case number!");
200 }
201 let opts = argv || {};
202 this.caseNum = caseNumber;
203 this.debug = opts.debug;
204 };
205
206 findCase(cb) {
207 let host = "odysseyportal.harriscountytx.gov";
208 let hc = new HTTPConversation();
209
210 hc.getURL({ reqHost: host, reqPath: "/OdysseyPortalJP", callback: () => {
211 hc.getURL({ reqHost: host, reqPath: "/OdysseyPortalJP/Home/Dashboard/29", callback: () => {
212 hc.postURLData({
213 reqHost: host,
214 reqPath: "/OdysseyPortalJP/SmartSearch/SmartSearch/SmartSearch",
215 data: [
216 [ "Settings.CaptchaEnabled", "False" ],
217 [ "Settings.CaptchaDisabledForAuthenticated", "False" ],
218 [ "caseCriteria.SearchCriteria", this.caseNum ],
219 [ "caseCriteria.JudicialOfficerSearchBy", "" ],
220 [ "caseCriteria.NameMiddle", "" ],
221 [ "caseCriteria.NameSuffix", "" ],
222 [ "caseCriteria.AdvancedSearchOptionsOpen", "false" ],
223 [ "caseCriteria.CourtLocation_input", "Harris County JPs Odyssey Portal" ],
224 [ "caseCriteria.CourtLocation", "Harris County JPs Odyssey Portal" ],
225 [ "caseCriteria.SearchBy_input", "Smart Search" ],
226 [ "caseCriteria.SearchBy", "SmartSearch" ],
227 [ "caseCriteria.SearchCases", "true" ],
228 [ "caseCriteria.SearchCases", "false" ],
229 [ "caseCriteria.SearchJudgments", "true" ],
230 [ "caseCriteria.SearchJudgments", "false" ],
231 [ "caseCriteria.SearchByPartyName", "true" ],
232 [ "caseCriteria.SearchByNickName", "false" ],
233 [ "caseCriteria.SearchByBusinessName", "false" ],
234 [ "caseCriteria.UseSoundex", "false" ],
235 [ "caseCriteria.Gender_input", "" ],
236 [ "caseCriteria.Gender", "" ],
237 [ "caseCriteria.Race_input", "" ],
238 [ "caseCriteria.Race", "" ],
239 [ "caseCriteria.FBINumber", "" ],
240 [ "caseCriteria.SONumber", "" ],
241 [ "caseCriteria.BookingNumber", "" ],
242 [ "caseCriteria.CaseType_input", "" ],
243 [ "caseCriteria.CaseType", "" ],
244 [ "caseCriteria.CaseStatus_input", "" ],
245 [ "caseCriteria.CaseStatus", "" ],
246 [ "caseCriteria.FileDateStart", "" ],
247 [ "caseCriteria.FileDateEnd", "" ],
248 [ "caseCriteria.JudicialOfficer_input", "" ],
249 [ "caseCriteria.JudicialOfficer", "" ],
250 [ "caseCriteria.JudgmentType_input", "" ],
251 [ "caseCriteria.JudgmentType", "" ],
252 [ "caseCriteria.JudgmentDateFrom", "" ],
253 [ "caseCriteria.JudgmentDateTo", "" ]
254 ],
255 callback: () => {
256 hc.getURL({ reqHost: host, reqPath: "/OdysseyPortalJP/Home/WorkspaceMode?p=0", callback: () => {
257 setTimeout(() => {
258 hc.getURL({
259 reqHost: host,
260 reqPath: "/OdysseyPortalJP/SmartSearch/SmartSearchResults?_="+(new Date()).valueOf(),
261 parser: getCaseIDParser(this, () => {
262 this.debug && console.log("CaseID: ", this.caseID);
263 hc.getURL({
264 reqHost: host,
265 reqPath: "/OdysseyPortalJP/Case/CaseDetail?eid="+this.caseID+"&tabIndex=3&_="+(new Date()).valueOf(),
266 parser: getCaseInfoParser(this, (info) => {
267 if (cb && "function" === typeof cb) {
268 cb(this);
269 } else {
270 console.log(this);
271 }
272 })
273 });
274 })
275 });
276 }, 1000);
277 }});
278 }});
279 }});
280 }});
281 };
282 };
File lib/httpOptions.js copied from file Promise.shim.js (similarity 53%) (mode: 100644) (index 354a297..097a8a1)
16 16
17 17 Copyright 2020 Luigi Bai Copyright 2020 Luigi Bai
18 18 */ */
19 module.exports = (promiseArr) => {
20 let pA =
21 Promise.all(
22 promiseArr.map(promise => {
23 if (promise instanceof Promise) {
24 return promise
25 .then(val => {
26 return { status: "fulfilled", value: val };
27 })
28 .catch(err => {
29 return { status: "rejected", reason: err };
30 })
31 ;
32 } else {
33 return { status: "fulfilled", value: promise };
34 }
35 })
36 )
37 ;
38 return pA;
19 // Keep a consistent set of connection options. Can be overridden.
20 module.exports = class {
21 static getGlobalAgent() {
22 if (!this.ga) this.gAgent = new (require("https")).Agent({
23 keepAlive: true,
24 maxSockets: 4
25 });;
26 return this.gAgent;
27 };
28
29 constructor() {
30 this.headers = {
31 "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0",
32 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
33 "Accept-Language": "en-US,en;q=0.5",
34 "Accept-Encoding": "identity"
35 };
36 this.method = "GET";
37 this.protocol = "https:";
38 this.agent = module.exports.getGlobalAgent();
39 };
39 40 }; };
File lib/sqlDAO.js changed (mode: 100644) (index 3551a66..324d944)
... ... let DAO = class {
26 26
27 27 finalize() { finalize() {
28 28 for (var s in this.stmts) { for (var s in this.stmts) {
29 console.log("Closing statement ", s);
30 29 this.stmts[s].finalize(); this.stmts[s].finalize();
31 30 } }
32 31 }; };
File loadDockets.js added (mode: 100644) (index 0000000..a0eb075)
1 /*
2 This file is part of datajams-evictions.
3
4 datajams-evictions is free software: you can redistribute it and/or modify
5 it under the terms of the GNU Affero General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 datajams-evictions is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU Affero General Public License for more details.
13
14 You should have received a copy of the GNU Affero General Public License
15 along with datajams-evictions. If not, see <https://www.gnu.org/licenses/>.
16
17 Copyright 2020 Luigi Bai
18 */
19 const DAO = require("./lib/sqlDAO");
20 const Court = require("./lib/Court.class");
21
22 let today = new Date();
23 let future = (new Date(today.valueOf()));
24 future.setDate(today.getDate() + 6);
25
26 // Emit the CSV headers; we get a line for each case, below:
27 console.log(
28 '"Case Number",',
29 '"Precinct",',
30 '"Place",',
31 '"Date",',
32 '"Time",',
33 '"DateTime",',
34 '"Claim",',
35 '"Plaintiff",',
36 '"Defendant 1",',
37 '"Defendant 2",',
38 '"Defendant 3",',
39 '"Case URL",',
40 '"Docket URL"'
41 );
42
43 let opts = require("./creds")["SQLITE3"];
44
45 opts.connectCallback = (database) => {
46 let promises = [];
47
48 // Loop over all 8 precincts, two places per precinct:
49 for (let pct = 1; pct <= 8; pct ++) {
50 for (let plc = 1; plc <= 2; plc ++) {
51 // param is pct * 10 + plc:
52 let court = new Court({
53 precinct: pct,
54 place: plc
55 });
56
57 promises.push(court.getDockets({
58 startDate: today,
59 endDate: future,
60 docketType: "Eviction Docket",
61 caseCallback: (caseObj) => {
62 // First, store the case into the database:
63 caseObj.storeSQL(database);
64
65 // These functions allow the "pivot" below on each case:
66 let plaintiffs = caseObj.parties.filter((party) => {
67 return ("Plaintiff" === party.role);
68 });
69 let defendants = caseObj.parties.filter((party) => {
70 return ("Defendant" === party.role);
71 });
72
73 // See the CSV header above? Emit each case to that format:
74 console.log(
75 caseObj.caseNumber + ",",
76 caseObj.docket.court.precinct + ",",
77 caseObj.docket.court.place + ",",
78 caseObj.docket.date + ",",
79 caseObj.docket.time + ",",
80 '"' +
81 caseObj.docket.date.substr(6)+"-"+
82 caseObj.docket.date.substr(0, 2)+"-"+
83 caseObj.docket.date.substr(3, 2)+" "+
84 caseObj.docket.time
85 + '",',
86 '"' + caseObj.claimAmount + '",',
87 (plaintiffs.length > 0) ? '"' + plaintiffs[0].name + '",' : '"",',
88 (defendants.length > 0) ? '"' + defendants[0].name + '",' : '"",',
89 (defendants.length > 1) ? '"' + defendants[1].name + '",' : '"",',
90 (defendants.length > 2) ? '"' + defendants[2].name + '",' : '"",',
91 '"' + caseObj.URL + '",',
92 '"' + caseObj.docket.URL.href + '",'
93 );
94 }
95 }));
96 }
97 }
98
99 // Wait for all the dockets to load, then emit their info:
100 Promise.all(promises)
101 .then(val => { console.log("All courts: ", val); })
102 .catch(err => { console.error("All courts ERR: ", err); })
103 .finally(() => {
104 console.log("Finished loading all dockets from all courts.");
105 (require("./lib/httpOptions").getGlobalAgent().destroy());
106 DAO.disconnect(database);
107 });
108 ;
109 };
110
111 DAO.connect(opts);
File schema.mssql.sql changed (mode: 100644) (index db57f26..3145b10)
... ... CREATE TABLE docket (
3 3 precinct INTEGER, precinct INTEGER,
4 4 place INTEGER, place INTEGER,
5 5 docket_dateTime datetime, docket_dateTime datetime,
6 URL nvarchar(1024) COLLATE Latin1_General_100_CS_AS_SC_UTF8,
6 URL nvarchar(1024) COLLATE SQL_Latin1_General_CP1250_CS_AS,
7 7 docket_ID INTEGER IDENTITY(1, 1) PRIMARY KEY docket_ID INTEGER IDENTITY(1, 1) PRIMARY KEY
8 8 ); );
9 9 DROP TABLE cases; DROP TABLE cases;
File searchNames.js added (mode: 100644) (index 0000000..20c3820)
1 const DAO = require("./lib/sqlDAO");
2 const Case = require("./lib/Case.class.js");
3
4 if (process.argv.length < 3) throw "Must pass in search string.";
5 let search = process.argv[2];
6
7 if (search.length < 8) throw "Search string must be at least 8 characters.";
8
9 let opts = require("./creds")["SQLITE3"];
10
11 opts.connectCallback = (database) => {
12 Case.findCasesByBusiness(search, (cases) => {
13 cases.forEach((caseUrl) => {
14 let caseObj = new Case(null, caseUrl);
15 let dao = new DAO.DAO();
16 dao.stmts.update = dao.prepare(
17 database,
18 "UPDATE cases SET filedDate = $fd, status = $status WHERE casenumber = $cnum"
19 );
20 caseObj.loadCaseFromURL((caseO) => {
21 caseO.storeSQL(database);
22 dao.update({
23 $cnum: caseO.caseNumber,
24 $fd: caseO.filedDate,
25 $status: caseO.caseStatus
26 });
27 });
28 });
29 });
30 };
31
32 DAO.connect(opts);
File sqdump added (mode: 100755) (index 0000000..7c25a48)
1 #!/bin/bash
2
3 if [ -z "$1" ]; then
4 echo "Must have a table name."
5 else
6 sqlite3 eviction-data.sq3 <<END
7 .headers on
8 .separator ,
9 select * from ${1};
10 .quit
11 END
12 fi
File updateStatus.js added (mode: 100644) (index 0000000..3baa858)
1 /*
2 This file is part of datajams-evictions.
3
4 datajams-evictions is free software: you can redistribute it and/or modify
5 it under the terms of the GNU Affero General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 datajams-evictions is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU Affero General Public License for more details.
13
14 You should have received a copy of the GNU Affero General Public License
15 along with datajams-evictions. If not, see <https://www.gnu.org/licenses/>.
16
17 Copyright 2020 Luigi Bai
18 */
19 const DAO = require("./lib/sqlDAO");
20 const Case = require("./lib/Case.class.js");
21
22 let opts = require("./creds")["SQLITE3"];
23 opts.connectCallback = (database) => {
24 // Prep the databse to update each case as we get its updated info:
25 let dao = new DAO.CaseCtlr(database);
26 dao.stmts.update = dao.prepare(
27 database,
28 "UPDATE cases SET filed_Date = $fd, case_status = $status WHERE casenumber = $cnum"
29 );
30
31 // Track all the update promises, then wait until they're done:
32 let promises = [];
33 Case.sqlAllCases(
34 database,
35 // This is the callback used for each case:
36 caseObj => {
37 promises.push(
38 // Load the data from the website, then
39 caseObj.loadCaseFromURL()
40 .then(caseO => {
41 // store it into the database:
42 caseO.storeSQL(database);
43 (caseO.caseNumber && caseO.filedDate && caseO.caseStatus) &&
44 dao.update({
45 $cnum: caseO.caseNumber,
46 $fd: caseO.filedDate,
47 $status: caseO.caseStatus
48 });
49 })
50 .catch(e => { console.error("loadCaseFromURL", caseObj, e); })
51 );
52 },
53 () => {
54 // And this is the callback used when the SELECT is finished:
55 Promise.all(promises)
56 .finally(() => {
57 // Shut down the database, and the HTTPS agent:
58 require("./lib/httpOptions").getGlobalAgent().destroy();
59 dao.finalize();
60 DAO.disconnect(database);
61 })
62 ;
63 }
64 );
65 };
66
67 DAO.connect(opts);
Hints:
Before first commit, do not forget to setup your git environment:
git config --global user.name "your_name_here"
git config --global user.email "your@email_here"

Clone this repository using HTTP(S):
git clone https://rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using ssh (do not forget to upload a key first):
git clone ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using git:
git clone git://git.rocketgit.com/user/datajams-lbai/datajams-evictions

You are allowed to anonymously push to this repository.
This means that your pushed commits will automatically be transformed into a merge request:
... clone the repository ...
... make some changes and some commits ...
git push origin main