datajams-lbai / datajams-evictions (public) (License: AGPLv3) (since 2020-09-01) (hash sha1)
Extracting Harris County evictions information for Houston DataJams
Clone URLs: https://rocketgit.com/user/datajams-lbai/datajams-evictions ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions git://git.rocketgit.com/user/datajams-lbai/datajams-evictions
first
List of commits:
Subject Hash Author Date (UTC)
Change to using TaskManager interface. cae71777479af7194082556c00be37d15c4e2537 Luigi Bai 2020-09-24 00:48:34
Actually care more about reaching recent filed_date cases than casenumber. Often they'll be pretty close, but casenumber will sort by precinct/place instead of filing date. 8d5f1c19cdf405281ceb9aa254c142d890da20a4 Luigi Bai 2020-09-21 17:36:26
Add some more date munging to get YYYY-MM-DD 40aaac1813b4dfa2ea2987027157cea491f81d2d Luigi Bai 2020-09-21 17:35:17
Try to abort earlier when connections fail. 32beffa93cc86e9c09101ab8637db07e8f5a2afb Luigi Bai 2020-09-21 17:34:59
Do insert before update on updateAddress in case party isn't there. Allow sqlDAO to ignore insert errors silently. 4bec38299c66cdf2cbb2317babde3c08dcd02ae1 Luigi Bai 2020-09-19 16:25:47
Allow passing in number of cases to update. ee680faa663867a52b2cbd49b68ba477f17c8cf0 Luigi Bai 2020-09-19 16:24:57
Give a heads up on how many cases will be udpated. 6624a0e47f1a2f326e4f079a93dc8beea84aae24 Luigi Bai 2020-09-18 21:37:32
Choose only Active cases or cases with NULLs to update. eadbf12cdc696dbe027578b2c352a09a2c6778b4 Luigi Bai 2020-09-18 21:37:13
Fix typo. 021fde3ca5c6697ea1c7c9cee00d4c2937fb6385 Luigi Bai 2020-09-18 21:36:36
Fix dates to YYYY-MM-DD on the way in, as much as possible. fa16c80ea14bad924f75ecc81d3cbfc4b2fe21e4 Luigi Bai 2020-09-18 21:36:20
Include some SQL scripts to update dates from the courts to YYYY-MM-DD. bde2825badffc0900955df5119107f38121eb14a Luigi Bai 2020-09-18 21:35:23
* improve date manipulation, replace moment with dayjs * (bug) Court.class extract parser now creates correct Docket and DocketedCase * don't bother with EXTRACT_HEARINGS, those are always empty * updateStatus updates status AND filed_Date 8ae2890852f5cb53c0429cc791098fee86f5b793 Luigi Bai 2020-09-18 02:49:03
Store case status and filedDate when available e0d7a3d44e4560e04133e777bdc0697bec57a8d4 Luigi Bai 2020-09-16 20:38:15
Clean up data xfer. Use TRUNCATE TABLE instead of DELETE FROM. e407788ccd1fb9ace1ca7cad4b0a1acd0189c799 Luigi Bai 2020-09-16 02:37:15
Clean up code. 3036e8ca177024179b3a6e391876121e526cd213 Luigi Bai 2020-09-16 02:36:42
Add bulkExtract importer. 0da4130e3e3a3e2e855f2ef28c4194fb6b5dfe10 Luigi Bai 2020-09-16 02:36:02
Add court related data 20f13f9ef9885794eddb2470bd57a9207dc7af20 Luigi Bai 2020-09-12 21:47:34
Try to avoid errors where information may be missing. 3410e49ba53bd7ebb417ee9bac7db1cf9e0e8808 Luigi Bai 2020-09-09 17:08:59
Don't put status information into console.log. 19dab8ff60a0256f9511391d4088c1b8f88ac3db Luigi Bai 2020-09-09 17:06:24
Add more info to README. 9708b396b887bab284e0ad2c2d996ea2db0ff4d4 Luigi Bai 2020-09-09 12:37:24
Commit cae71777479af7194082556c00be37d15c4e2537 - Change to using TaskManager interface.
Author: Luigi Bai
Author date (UTC): 2020-09-24 00:48
Committer name: Luigi Bai
Committer date (UTC): 2020-09-24 00:48
Parent(s): 8d5f1c19cdf405281ceb9aa254c142d890da20a4
Signing key:
Tree: 68a44428e516e07692a1741a2e6bf400efc0674a
File Lines added Lines deleted
lib/TaskManager.class.js 406 0
schema.sqlite3.sql 13 0
updateAddress.js 17 70
updateEvents.js 54 0
updateStatus.js 23 39
File lib/TaskManager.class.js added (mode: 100644) (index 0000000..92242e2)
1 /*
2 This file is part of datajams-evictions.
3
4 datajams-evictions is free software: you can redistribute it and/or modify
5 it under the terms of the GNU Affero General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 datajams-evictions is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU Affero General Public License for more details.
13
14 You should have received a copy of the GNU Affero General Public License
15 along with datajams-evictions. If not, see <https://www.gnu.org/licenses/>.
16
17 Copyright 2020 Luigi Bai
18 */
19 const DAO = require("./sqlDAO");
20 const m = require("dayjs");
21 const OI = require("./OdysseyInfo.class");
22 const Case = require("./Case.class");
23
24 let managers = {};
25 let tsFormat = "YYYY-MM-DDTHH:mm:ss.SSSZ";
26
27 class TaskManager {
28 constructor(database, name) {
29 if (managers[name]) {
30 throw "Already have a "+name;
31 }
32 this.name = name;
33 this.database = database;
34 this.started = {};
35 managers[name] = this;
36
37 this.insert = database.prepare("INSERT INTO tasks (name, data, queued, started) VALUES ($nm, $dt, $qd, 'N')");
38 this.start = database.prepare("UPDATE tasks SET started = $st WHERE name = $nm AND data = $dt AND started ='N'");
39 this.finish = database.prepare("UPDATE tasks SET ended = $et, success = $val, failure = $err WHERE name = $nm AND data = $dt AND started = $st");
40
41 };
42
43 finalize() {
44 this.insert.finalize();
45 this.start.finalize();
46 this.finish.finalize();
47 };
48 addTask(data) {
49 let dObj = {
50 $dt: JSON.stringify(data),
51 $qd: (new m()).format(tsFormat),
52 $nm: this.name
53 };
54
55 this.insert.run(
56 dObj,
57 e => {
58 if (e) {
59 switch (e.errno) {
60 case 19:
61 // Ignore insert errors.
62 break;
63 default:
64 throw { msg: "Insert task", err: e};
65 }
66 }
67 }
68 );
69 };
70 startTask(data) {
71 let now = (new m()).format(tsFormat);
72 let dts = JSON.stringify(data);
73 this.started[dts] = now;
74 this.start.run(
75 {
76 $st: now,
77 $dt: dts,
78 $nm: this.name
79 },
80 e => {
81 if (e) {
82 throw { msg: "Start task", err: e};
83 }
84 }
85 );
86 };
87 endTask(data, val, err) {
88 let now = (new m()).format(tsFormat);
89 let dts = JSON.stringify(data);
90 if (! this.started[dts]) throw { msg: "No task started", data: data };
91 this.finish.run(
92 {
93 $et: now,
94 $st: this.started[dts],
95 $dt: dts,
96 $nm: this.name,
97 $val: (val) ? JSON.stringify(val).substr(1, 4000) : null,
98 $err: (err) ? JSON.stringify(err).substr(1, 4000) : null
99 },
100 e => {
101 if (e) {
102 throw { msg: "Finish task", err: e};
103 }
104 }
105 );
106 this.started[dts] = null;
107 };
108 loadTasks() {
109 return TaskManager.loadTasks(this.name);
110 };
111 runTasks(rowLimit) {
112 return TaskManager.runTasks(this.name, rowLimit);
113 };
114
115 static runTask(name, data) {
116 return new Promise((resFN, rejFN) => {
117 if (!managers[name]) {
118 rejFN({ msg: "Run "+name, data: data, err: "No such task defined." });
119 } else {
120 if (!managers[name].runTask) {
121 rejFN({ msg: "Run "+name, data: data, err: "No runTask defined." });
122 }
123 }
124 try {
125 managers[name].startTask(data);
126 managers[name].runTask(data)
127 .then(val => {
128 try {
129 managers[name].endTask(data, val, null);
130 } catch (e) {
131 console.error("End task", e);
132 }
133 resFN(val);
134 })
135 .catch(err => {
136 try {
137 managers[name].endTask(data, null, err);
138 } catch (e) {
139 console.error("End task", e);
140 }
141 rejFN({ msg: "Run "+name, data: data, err: err });
142 })
143 ;
144 } catch (e) {
145 rejFN({ msg: "Start "+name, data: data, err: e });
146 }
147 });
148 };
149 static loadTasks(name) {
150 return new Promise((resFN, rejFN) => {
151 if (!managers[name]) {
152 rejFN({ msg: "LoadTasks "+name, err: "No such task defined." });
153 }
154
155 let rowCount = 0;
156 managers[name].database.each(
157 managers[name].getTaskSelectSQL(),
158 (err, row) => {
159 if (err) {
160 console.error("add tasks", err);
161 } else {
162 rowCount++;
163 managers[name].addTask(row);
164 }
165 },
166 () => {
167 resFN(rowCount);
168 }
169 );
170 });
171 };
172 static runTasks(name, num) {
173 return new Promise((resFN, rejFN) => {
174 if (!managers[name]) {
175 rejFN({ msg: "RunTasks "+name, err: "No such task defined." });
176 } else {
177 if (!managers[name].runTask) {
178 rejFN({ msg: "RunTasks "+name, err: "No runTask defined." });
179 }
180 }
181 let promises = [];
182
183 let limit = 25;
184 if (!Number.isNaN(num) && Number(num).valueOf() > 0)
185 limit = num;
186 console.log("Queueing up to", limit, "tasks for", name);
187
188 managers[name].database.each(
189 "SELECT name, data FROM tasks WHERE name=$nm AND started = 'N' ORDER BY queued DESC LIMIT "+limit,
190 { $nm: name },
191 (err, row) => {
192 if (err) {
193 rejFN({ msg: "Run tasks "+name, err: err });
194 } else {
195 promises.push(
196 TaskManager.runTask(name, JSON.parse(row.data))
197 );
198 }
199 },
200 () => {
201 console.log("Running", promises.length, name, "tasks.");
202 Promise.all(promises)
203 .then(val => { resFN(val); })
204 .catch(err => { rejFN(err); })
205 ;
206 }
207 );
208 });
209 };
210 };
211
212 module.exports = {
213 updateAddresses: class extends TaskManager {
214 constructor(database) {
215 super(database, "updateAddresses");
216
217 // Prep the databse to update each case as we get its updated info:
218 this.pDao = new DAO.PartyCtlr(database);
219 this.pDao.stmts.update = this.pDao.prepare(
220 database,
221 "UPDATE party SET party_address = $pa WHERE casenumber = $cnum AND party_role = $pr AND party_name = $pn"
222 );
223 this.pDao.ignoreInsertErrors = true;
224 this.cDao = new DAO.CaseCtlr(database);
225 this.cDao.stmts.update = this.cDao.prepare(
226 database,
227 "UPDATE cases SET odyssey_ID = $oid WHERE casenumber = $cnum"
228 );
229 };
230
231 getTaskSelectSQL() {
232 let sql = "\
233 SELECT casenumber as caseNumber FROM cases \
234 WHERE casenumber IS NOT NULL \
235 AND odyssey_ID IS NULL \
236 AND casenumber NOT IN ( \
237 SELECT json_extract(data, '$.caseNumber') FROM tasks \
238 WHERE name = 'updateAddresses' AND started = 'N' \
239 ) \
240 "
241 ;
242 return sql;
243 };
244
245 finalize() {
246 this.pDao.finalize();
247 this.cDao.finalize();
248 super.finalize();
249 };
250
251 // MUST RETURN A PROMISE
252 runTask(data) {
253 if (! data.caseNumber)
254 console.error("data with no casenumber?");
255
256 return (new OI(data.caseNumber))
257 .findCasePromise()
258 .then(oi => {
259 // Store the OdysseyID into Cases
260 this.cDao.update({
261 $cnum: oi.caseNumber,
262 $oid: oi.caseID
263 });
264 // Store the party addresses
265 oi.parties.forEach(party => {
266 if (party.address) {
267 // If this has been a bulk extract,
268 // probably should insert first, given
269 // the bulk extracts only have 2 parties
270 // per case.
271 this.pDao.insert({
272 $addr: party.address.join(" "),
273 $cnum: oi.caseNumber,
274 $role: party.role,
275 $name: party.name
276 });
277 this.pDao.update({
278 $pa: party.address.join(" "),
279 $cnum: oi.caseNumber,
280 $pr: party.role,
281 $pn: party.name
282 });
283 }
284 });
285 // This gets logged into the database:
286 return { msg: "OI stored", case: data.caseNumber };
287 })
288 .catch(err => {
289 // This get logged into the database:
290 return { msg: "Error retrieving OI", case: data.caseNumber, err: err };
291 })
292 ;
293 };
294 },
295
296 updateEvents: class extends TaskManager {
297 constructor(database) {
298 super(database, "updateEvents");
299 };
300
301 getTaskSelectSQL() {
302 let sql = "\
303 SELECT cases.casenumber as caseNumber, coalesce(ev.num, 0) as evCount \
304 FROM cases LEFT OUTER JOIN (select events.casenumber, count(*) num from events group by events.casenumber) ev \
305 ON cases.casenumber = ev.casenumber \
306 WHERE evCount between 0 and 1 \
307 AND casenumber NOT IN ( \
308 SELECT json_extract(data, '$.caseNumber') FROM tasks \
309 WHERE name = 'updateEvents' AND started = 'N' \
310 ) \
311 "
312 ;
313
314 return sql;
315 };
316
317 // MUST RETURN A PROMISE
318 runTask(data) {
319 if (! data.caseNumber)
320 console.error("data with no casenumber?");
321 return (new Case(null, null, data.caseNumber))
322 .loadCaseFromURL()
323 .then(cObj => {
324 cObj.storeSQL(this.database);
325 // This gets logged into the database:
326 return { msg: "Events stored", case: data.caseNumber };
327 })
328 .catch(err => {
329 // This gets logged into the database:
330 return { msg: "Error storing events", case: data.caseNumber, err: err };
331 })
332 ;
333 };
334 },
335
336 updateStatus: class extends TaskManager {
337 constructor(database) {
338 super(database, "updateStatus");
339
340 // Prep the databse to update each case as we get its updated info:
341 this.cDao = new DAO.CaseCtlr(database);
342 this.cDao.stmts.update = this.cDao.prepare(
343 database,
344 "UPDATE cases SET filed_Date = $fd, case_status = $status WHERE casenumber = $cnum"
345 );
346 };
347
348 getTaskSelectSQL() {
349 let sql = " \
350 SELECT case_URL FROM cases \
351 WHERE ( \
352 case_status IS NULL \
353 OR filed_date IS NULL \
354 OR ( \
355 case_status = 'Active' \
356 AND casenumber NOT IN ( \
357 SELECT casenumber FROM events \
358 WHERE dateAdded > datetime('now', '-1 day') \
359 ) \
360 ) \
361 ) AND ( \
362 case_URL NOT IN ( \
363 SELECT json_extract(data, '$.case_URL') FROM tasks \
364 WHERE name = 'updateStatus' \
365 AND ( \
366 ended IS NULL OR \
367 DATE(ended) = DATE('now') \
368 )\
369 ) \
370 )"
371 ;
372 return sql;
373 };
374
375 finalize() {
376 this.cDao.finalize();
377 super.finalize();
378 };
379
380 // MUST RETURN A PROMISE
381 runTask(data) {
382 if (! data["case_URL"])
383 console.error("data with no case URL?");
384
385 let caseObj = new Case(null, data["case_URL"]);
386 return caseObj.loadCaseFromURL()
387 .then(caseO => {
388 // store it into the database:
389 caseO.storeSQL(this.database);
390 (caseO.caseNumber && caseO.filedDate && caseO.caseStatus) &&
391 this.cDao.update({
392 $cnum: caseO.caseNumber,
393 $fd: (new m(caseO.filedDate).format("YYYY-MM-DD")),
394 $status: caseO.caseStatus
395 });
396 // This gets logged into the database:
397 return { msg: "Status stored", case: caseObj.caseNumber };
398 })
399 .catch(err => {
400 // This gets logged into the database:
401 return { msg: "Error storing status", case: caseObj.caseNumber, err: err };
402 })
403 ;
404 };
405 }
406 };
File schema.sqlite3.sql changed (mode: 100644) (index a05db94..062c3cb)
... ... CREATE TABLE events (
70 70 eventID INTEGER PRIMARY KEY eventID INTEGER PRIMARY KEY
71 71 ); );
72 72
73 DROP TABLE tasks;
74 CREATE TABLE tasks (
75 name NVARCHAR(1024) NOT NULL,
76 data NVARCHAR(4000) NOT NULL,
77 queued NVARCHAR(64) NOT NULL,
78 started NVARCHAR(64),
79 ended NVARCHAR(64),
80 success NVARCHAR(4000),
81 failure NVARCHAR(4000),
82 taskID INTEGER PRIMARY KEY
83 );
84 CREATE UNIQUE INDEX taskIDX on tasks (name, data, started);
85
73 86 CREATE UNIQUE INDEX docketTime on docket (precinct, place, docket_dateTime); CREATE UNIQUE INDEX docketTime on docket (precinct, place, docket_dateTime);
74 87 CREATE UNIQUE INDEX docketedCase on docketedCases (precinct, place, docket_dateTime, casenumber); CREATE UNIQUE INDEX docketedCase on docketedCases (precinct, place, docket_dateTime, casenumber);
75 88 CREATE UNIQUE INDEX casenum on cases (casenumber); CREATE UNIQUE INDEX casenum on cases (casenumber);
File updateAddress.js changed (mode: 100644) (index 432e290..b50d6a4)
17 17 Copyright 2020 Luigi Bai Copyright 2020 Luigi Bai
18 18 */ */
19 19 const DAO = require("./lib/sqlDAO"); const DAO = require("./lib/sqlDAO");
20 const Case = require("./lib/Case.class");
21 const OdysseyInfo = require("./lib/OdysseyInfo.class");
20 const TM = require("./lib/TaskManager.class");
21 const taskName = "updateAddresses";
22 22
23 23 let rowLimit = 25; let rowLimit = 25;
24 24 if (process.argv.length > 2) { if (process.argv.length > 2) {
 
... ... if (process.argv.length > 2) {
28 28
29 29 let opts = require("./creds")["SQLITE3"]; let opts = require("./creds")["SQLITE3"];
30 30 opts.connectCallback = database => { opts.connectCallback = database => {
31 // Prep the databse to update each case as we get its updated info:
32 let pDao = new DAO.PartyCtlr(database);
33 pDao.stmts.update = pDao.prepare(
34 database,
35 "UPDATE party SET party_address = $pa WHERE casenumber = $cnum AND party_role = $pr AND party_name = $pn"
36 );
37 pDao.ignoreInsertErrors = true;
38 let cDao = new DAO.CaseCtlr(database);
39 cDao.stmts.update = cDao.prepare(
40 database,
41 "UPDATE cases SET odyssey_ID = $oid WHERE casenumber = $cnum"
42 );
43
44
45 // Track all the update promises, then wait until they're done:
46 let promises = [];
47 database.each(
48 "SELECT casenumber as caseNumber FROM cases WHERE casenumber IS NOT NULL AND odyssey_ID IS NULL ORDER BY filed_date DESC LIMIT "+rowLimit,
49 // This is the callback used for each case:
50 (err, caseObj) => {
51 if (! caseObj.caseNumber) console.error("Database had case with no casenuber.");
52 else {
53 promises.push(
54 // Grab the
55 (new OdysseyInfo(caseObj.caseNumber))
56 .findCasePromise()
57 .then(oi => {
58 // Store the OdysseyID into Cases
59 cDao.update({
60 $cnum: oi.caseNumber,
61 $oid: oi.caseID
62 });
63 // Store the party addresses
64 oi.parties.forEach(party => {
65 if (party.address) {
66 // If this has been a bulk extract,
67 // probably should insert first, given
68 // the bulk extracts only have 2 parties
69 // per case.
70 pDao.insert({
71 $addr: party.address.join(" "),
72 $cnum: oi.caseNumber,
73 $role: party.role,
74 $name: party.name
75 });
76 pDao.update({
77 $pa: party.address.join(" "),
78 $cnum: oi.caseNumber,
79 $pr: party.role,
80 $pn: party.name
81 });
82 }
83 });
84 })
85 .catch(err => {
86 console.error("Error retrieving OI", caseObj, err);
87 })
88 );
89 }
90 },
91 () => {
92 console.log("Updating", promises.length, "addresses.");
93 // And this is the callback used when the SELECT is finished:
94 Promise.all(promises)
31 let UATask = new (TM[taskName])(database);
32
33 UATask.loadTasks(taskName)
34 .then(rowCount => {
35 // Now run the tasks, given the SELECT/store is finished:
36 console.log("Stored", rowCount, "new tasks.");
37 UATask.runTasks(taskName, rowLimit)
38 .then(val => { console.log("Done."); })
39 .catch(err => { console.log("Error", err); })
95 40 .finally(() => { .finally(() => {
96 41 // Shut down the database, and the HTTPS agent: // Shut down the database, and the HTTPS agent:
97 pDao.finalize();
98 cDao.finalize();
42 UATask.finalize();
99 43 DAO.disconnect(database); DAO.disconnect(database);
100 44 require("./lib/httpOptions").getGlobalAgent().destroy(); require("./lib/httpOptions").getGlobalAgent().destroy();
101 45 }) })
102 46 ; ;
103 }
104 );
47 })
48 .catch(err => {
49 console.error("Error loading tasks", err);
50 })
51 ;
105 52 }; };
106 53
107 54 DAO.connect(opts); DAO.connect(opts);
File updateEvents.js added (mode: 100644) (index 0000000..1422750)
1 /*
2 This file is part of datajams-evictions.
3
4 datajams-evictions is free software: you can redistribute it and/or modify
5 it under the terms of the GNU Affero General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 datajams-evictions is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU Affero General Public License for more details.
13
14 You should have received a copy of the GNU Affero General Public License
15 along with datajams-evictions. If not, see <https://www.gnu.org/licenses/>.
16
17 Copyright 2020 Luigi Bai
18 */
19 const DAO = require("./lib/sqlDAO");
20 const TM = require("./lib/TaskManager.class");
21 const taskName = "updateEvents";
22
23 let rowLimit = 25;
24 if (process.argv.length > 2) {
25 let n = Number(process.argv[2]);
26 if (!Number.isNaN(n) && n.valueOf() > 0) rowLimit = n.valueOf();
27 }
28
29 let opts = require("./creds")["SQLITE3"];
30 opts.connectCallback = database => {
31 let UETask = new (TM[taskName])(database);
32
33 UETask.loadTasks()
34 .then(rowCount => {
35 // Now run the tasks, given the SELECT/store is finished:
36 console.log("Stored", rowCount, "new tasks.");
37 UETask.runTasks(rowLimit)
38 .then(val => { console.log("Done."); })
39 .catch(err => { console.log("Error", err); })
40 .finally(() => {
41 // Shut down the database, and the HTTPS agent:
42 UETask.finalize();
43 DAO.disconnect(database);
44 require("./lib/httpOptions").getGlobalAgent().destroy();
45 })
46 ;
47 })
48 .catch(err => {
49 console.error("Error loading tasks", err);
50 })
51 ;
52 };
53
54 DAO.connect(opts);
File updateStatus.js changed (mode: 100644) (index 99b21aa..173e2db)
17 17 Copyright 2020 Luigi Bai Copyright 2020 Luigi Bai
18 18 */ */
19 19 const DAO = require("./lib/sqlDAO"); const DAO = require("./lib/sqlDAO");
20 const Case = require("./lib/Case.class.js");
21 const m=require("dayjs");
20 const TM = require("./lib/TaskManager.class");
21 const taskName = "updateStatus";
22
23 let rowLimit = 100;
24 if (process.argv.length > 2) {
25 let n = Number(process.argv[2]);
26 if (!Number.isNaN(n) && n.valueOf() > 0) rowLimit = n.valueOf();
27 }
22 28
23 29 let opts = require("./creds")["SQLITE3"]; let opts = require("./creds")["SQLITE3"];
24 30 opts.connectCallback = (database) => { opts.connectCallback = (database) => {
25 // Prep the databse to update each case as we get its updated info:
26 let dao = new DAO.CaseCtlr(database);
27 dao.stmts.update = dao.prepare(
28 database,
29 "UPDATE cases SET filed_Date = $fd, case_status = $status WHERE casenumber = $cnum"
30 );
31 let USTask = new (TM[taskName])(database);
31 32
32 // Track all the update promises, then wait until they're done:
33 let promises = [];
34 Case.sqlAllCases({
35 database: database,
36 query: "SELECT case_URL FROM cases WHERE case_status IS NULL OR case_status = 'Active' OR filed_Date IS NULL LIMIT 100",
37 // This is the callback used for each case:
38 rowCallback: caseObj => {
39 promises.push(
40 // Load the data from the website, then
41 caseObj.loadCaseFromURL()
42 .then(caseO => {
43 // store it into the database:
44 caseO.storeSQL(database);
45 (caseO.caseNumber && caseO.filedDate && caseO.caseStatus) &&
46 dao.update({
47 $cnum: caseO.caseNumber,
48 $fd: (new m(caseO.filedDate).format("YYYY-MM-DD")),
49 $status: caseO.caseStatus
50 });
51 })
52 .catch(e => { console.error("loadCaseFromURL", caseObj, e); })
53 );
54 },
55 completionCallback: () => {
56 console.log("Updating", promises.length, "cases.");
57 // And this is the callback used when the SELECT is finished:
58 Promise.all(promises)
33 USTask.loadTasks()
34 .then(rowCount => {
35 // Now run the tasks, given the SELECT/store is finished:
36 console.log("Stored", rowCount, "new tasks.");
37 USTask.runTasks(rowLimit)
38 .then(val => { console.log("Done."); })
39 .catch(err => { console.log("Error", err); })
59 40 .finally(() => { .finally(() => {
60 41 // Shut down the database, and the HTTPS agent: // Shut down the database, and the HTTPS agent:
61 require("./lib/httpOptions").getGlobalAgent().destroy();
62 dao.finalize();
42 USTask.finalize();
63 43 DAO.disconnect(database); DAO.disconnect(database);
44 require("./lib/httpOptions").getGlobalAgent().destroy();
64 45 }) })
65 46 ; ;
66 }
67 });
47 })
48 .catch(err => {
49 console.error("Error loading tasks", err);
50 })
51 ;
68 52 }; };
69 53
70 54 DAO.connect(opts); DAO.connect(opts);
Hints:
Before first commit, do not forget to setup your git environment:
git config --global user.name "your_name_here"
git config --global user.email "your@email_here"

Clone this repository using HTTP(S):
git clone https://rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using ssh (do not forget to upload a key first):
git clone ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using git:
git clone git://git.rocketgit.com/user/datajams-lbai/datajams-evictions

You are allowed to anonymously push to this repository.
This means that your pushed commits will automatically be transformed into a merge request:
... clone the repository ...
... make some changes and some commits ...
git push origin main