datajams-lbai / datajams-evictions (public) (License: AGPLv3) (since 2020-09-01) (hash sha1)
Extracting Harris County evictions information for Houston DataJams
Clone URLs: https://rocketgit.com/user/datajams-lbai/datajams-evictions ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions git://git.rocketgit.com/user/datajams-lbai/datajams-evictions
first
List of commits:
Subject Hash Author Date (UTC)
* improve date manipulation, replace moment with dayjs * (bug) Court.class extract parser now creates correct Docket and DocketedCase * don't bother with EXTRACT_HEARINGS, those are always empty * updateStatus updates status AND filed_Date 8ae2890852f5cb53c0429cc791098fee86f5b793 Luigi Bai 2020-09-18 02:49:03
Store case status and filedDate when available e0d7a3d44e4560e04133e777bdc0697bec57a8d4 Luigi Bai 2020-09-16 20:38:15
Clean up data xfer. Use TRUNCATE TABLE instead of DELETE FROM. e407788ccd1fb9ace1ca7cad4b0a1acd0189c799 Luigi Bai 2020-09-16 02:37:15
Clean up code. 3036e8ca177024179b3a6e391876121e526cd213 Luigi Bai 2020-09-16 02:36:42
Add bulkExtract importer. 0da4130e3e3a3e2e855f2ef28c4194fb6b5dfe10 Luigi Bai 2020-09-16 02:36:02
Add court related data 20f13f9ef9885794eddb2470bd57a9207dc7af20 Luigi Bai 2020-09-12 21:47:34
Try to avoid errors where information may be missing. 3410e49ba53bd7ebb417ee9bac7db1cf9e0e8808 Luigi Bai 2020-09-09 17:08:59
Don't put status information into console.log. 19dab8ff60a0256f9511391d4088c1b8f88ac3db Luigi Bai 2020-09-09 17:06:24
Add more info to README. 9708b396b887bab284e0ad2c2d996ea2db0ff4d4 Luigi Bai 2020-09-09 12:37:24
Add info to README. 79672fad423a570d7eb05c4891dfabb511db5ab3 Luigi Bai 2020-09-09 00:20:23
Fixed searchNames. Now you can load cases where you know one of the parties. 0ad812704e27341a481a6de18c88a4f38bb61700 Luigi Bai 2020-09-08 23:43:48
Apparently updating addresses works, as long as you don't ask for more than 10 per minute. That seems to be working so far. 9276b2acb2cc3067c53fc16317a8960db3f6fcca Luigi Bai 2020-09-08 22:44:24
More information about how to use the software. Not much, yet. Enough to get started, I think. 393fce58bbcfeaee947cb2b0e17d81d16da49612 Luigi Bai 2020-09-08 22:31:02
Don't write to console.err if there's a rejectionFunction. 630b8ec76a21733a6bb93d98ca54563aadd61da4 Luigi Bai 2020-09-08 22:30:29
Remove trailing comma on all rows. e7605fa152a841eefc7fcab6bb7cfd64daf28b17 Luigi Bai 2020-09-07 22:43:36
Allow passing in query for Case.sqlAllCases() 346862c0720ef9054426048cc95caa8f6b068868 Luigi Bai 2020-09-07 22:43:09
This version works in MS-SQL. 4e3f33f464173c91c3f4ac4b6acb0235ed2d6720 Luigi Bai 2020-09-07 19:11:54
Add headers. 4da1ccbdb2e3e852050de3460bc9ac9ba061ed7a Luigi Bai 2020-09-07 16:28:52
loadDockets now works to local. 96ba8f7419f63be6612fe1efd3e24d5d30eaf20a Luigi Bai 2020-09-06 20:59:06
Fix for new column names. 5791d77e19585c494e51c6062ab45440e5af0903 Luigi Bai 2020-09-06 20:44:50
Commit 8ae2890852f5cb53c0429cc791098fee86f5b793 - * improve date manipulation, replace moment with dayjs * (bug) Court.class extract parser now creates correct Docket and DocketedCase * don't bother with EXTRACT_HEARINGS, those are always empty * updateStatus updates status AND filed_Date
Author: Luigi Bai
Author date (UTC): 2020-09-18 02:49
Committer name: Luigi Bai
Committer date (UTC): 2020-09-18 02:49
Parent(s): e0d7a3d44e4560e04133e777bdc0697bec57a8d4
Signing key:
Tree: 12f8c98728acb21c3bfc4ef1ad8f881e22201f94
File Lines added Lines deleted
extractMonth.js 2 2
lib/Court.class.js 8 8
package-lock.json 11 11
package.json 2 2
updateStatus.js 1 0
File extractMonth.js changed (mode: 100644) (index 0e61fd5..530f6b3)
18 18 */ */
19 19 const Court = require("./lib/Court.class"); const Court = require("./lib/Court.class");
20 20 const sqDAO = require("./lib/sqlDAO"); const sqDAO = require("./lib/sqlDAO");
21 const moment = require("moment");
21 const moment = require("dayjs");
22 22
23 23 if (process.argv.length < 4) throw "Must pass in YYYY and MM."; if (process.argv.length < 4) throw "Must pass in YYYY and MM.";
24 24 let year = process.argv[2]; let year = process.argv[2];
 
... ... opts.connectCallback = sqDB => {
38 38 precinct: pct, precinct: pct,
39 39 place: plc place: plc
40 40 }); });
41 for (var et of [Court.EXTRACT_FILED, Court.EXTRACT_HEARINGS, Court.EXTRACT_DISPOSED]) {
41 for (var et of [Court.EXTRACT_FILED, Court.EXTRACT_DISPOSED]) {
42 42 let exOpts = { let exOpts = {
43 43 database: sqDB, database: sqDB,
44 44 startDate: sDate.format("MM/DD/YYYY"), startDate: sDate.format("MM/DD/YYYY"),
File lib/Court.class.js changed (mode: 100644) (index ffb8fd8..4d9e1d0)
... ... const DomHandler = require("domhandler").DomHandler;
24 24 const du = require("domutils"); const du = require("domutils");
25 25 const Docket = require("./Docket.class"); const Docket = require("./Docket.class");
26 26 const Case = require("./Case.class"); const Case = require("./Case.class");
27 const moment = require("dayjs");
28 moment.extend(require("dayjs/plugin/customParseFormat"));
27 29
28 30 // Create parser: // Create parser:
29 31 let getExtractInfo = node => { let getExtractInfo = node => {
 
... ... let extractAddParty = (caseObj, row, pfx, role) => {
77 79
78 80 }; };
79 81
80 let extractParserGenerator = (resFN, rejFN, database) => {
82 let extractParserGenerator = (court, resFN, rejFN, database) => {
81 83 let casePromises = []; let casePromises = [];
82 84 return new hp.Parser( return new hp.Parser(
83 85 new DomHandler((err, dom) => { new DomHandler((err, dom) => {
 
... ... let extractParserGenerator = (resFN, rejFN, database) => {
145 147
146 148 let docket = null; let docket = null;
147 149 if (info.eventDate && info.eventTime) { if (info.eventDate && info.eventTime) {
148 let hours = info.eventTime.substr(0, 2);
149 let mins = info.eventTime.substr(3, 2);
150 let ap = info.eventTime.substr(6, 1);
150 let md = moment(info.eventTime, "hh:mm A");
151 151 docket = new Docket({ docket = new Docket({
152 court: this,
152 court: court,
153 153 date: info.eventDate, date: info.eventDate,
154 time: (hours + ("P" === ap) ? 12 : 0) + "" + mins + "00"
154 time: md.format("HHmm00")
155 155 }); });
156 156 } }
157 157
 
... ... module.exports = class Court {
213 213 } }
214 214
215 215 return new Promise((resFN, rejFN) => { return new Promise((resFN, rejFN) => {
216 let parser = extractParserGenerator(resFN, rejFN, argv.database);
216 let parser = extractParserGenerator(this, resFN, rejFN, argv.database);
217 217 argv.extractStream argv.extractStream
218 218 .on("end", () => { parser.done(); }) .on("end", () => { parser.done(); })
219 219 .on("data", d => { parser.parseChunk(d); }) .on("data", d => { parser.parseChunk(d); })
 
... ... module.exports = class Court {
252 252 rejFN({ msg: "court extractID: ", err: err }); rejFN({ msg: "court extractID: ", err: err });
253 253 } else { } else {
254 254 extractCourtID = row.extractID; extractCourtID = row.extractID;
255 let parser = extractParserGenerator(resFN, rejFN, argv.database);
255 let parser = extractParserGenerator(this, resFN, rejFN, argv.database);
256 256
257 257 // create the extractURL: // create the extractURL:
258 258 let extractURL = "https://jpwebsite.harriscountytx.gov/PublicExtracts/GetExtractData?extractCaseType=CV&extract="+argv.extractType+"&court="+extractCourtID+"&casetype=EV&format=xml&fdate="+argv.startDate+"&tdate="+argv.endDate; let extractURL = "https://jpwebsite.harriscountytx.gov/PublicExtracts/GetExtractData?extractCaseType=CV&extract="+argv.extractType+"&court="+extractCourtID+"&casetype=EV&format=xml&fdate="+argv.startDate+"&tdate="+argv.endDate;
File package-lock.json changed (mode: 100644) (index 7f7b560..4e6be3f)
290 290 "resolved": "https://registry.npmjs.org/date-utils/-/date-utils-1.2.21.tgz", "resolved": "https://registry.npmjs.org/date-utils/-/date-utils-1.2.21.tgz",
291 291 "integrity": "sha1-YfsWzcEnSzyayq/+n8ad+HIKK2Q=" "integrity": "sha1-YfsWzcEnSzyayq/+n8ad+HIKK2Q="
292 292 }, },
293 "dayjs": {
294 "version": "1.8.36",
295 "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.8.36.tgz",
296 "integrity": "sha512-3VmRXEtw7RZKAf+4Tv1Ym9AGeo8r8+CjDi26x+7SYQil1UqtqdaokhzoEJohqlzt0m5kacJSDhJQkG/LWhpRBw=="
297 },
293 298 "debug": { "debug": {
294 299 "version": "4.1.1", "version": "4.1.1",
295 300 "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
 
324 329 "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
325 330 }, },
326 331 "dom-serializer": { "dom-serializer": {
327 "version": "1.0.1",
328 "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.0.1.tgz",
329 "integrity": "sha512-1Aj1Qy3YLbdslkI75QEOfdp9TkQ3o8LRISAzxOibjBs/xWwr1WxZFOQphFkZuepHFGo+kB8e5FVJSS0faAJ4Rw==",
332 "version": "1.1.0",
333 "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.1.0.tgz",
334 "integrity": "sha512-ox7bvGXt2n+uLWtCRLybYx60IrOlWL/aCebWJk1T0d4m3y2tzf4U3ij9wBMUb6YJZpz06HCCYuyCDveE2xXmzQ==",
330 335 "requires": { "requires": {
331 336 "domelementtype": "^2.0.1", "domelementtype": "^2.0.1",
332 337 "domhandler": "^3.0.0", "domhandler": "^3.0.0",
 
347 352 } }
348 353 }, },
349 354 "domutils": { "domutils": {
350 "version": "2.2.0",
351 "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.2.0.tgz",
352 "integrity": "sha512-0haAxVr1PR0SqYwCH7mxMpHZUwjih9oPPedqpR/KufsnxPyZ9dyVw1R5093qnJF3WXSbjBkdzRWLw/knJV/fAg==",
355 "version": "2.3.0",
356 "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.3.0.tgz",
357 "integrity": "sha512-xWC75PM3QF6MjE5e58OzwTX0B/rPQnlqH0YyXB/c056RtVJA+eu60da2I/bdnEHzEYC00g8QaZUlAbqOZVbOsw==",
353 358 "requires": { "requires": {
354 359 "dom-serializer": "^1.0.1", "dom-serializer": "^1.0.1",
355 360 "domelementtype": "^2.0.1", "domelementtype": "^2.0.1",
 
713 718 "minimist": "^1.2.5" "minimist": "^1.2.5"
714 719 } }
715 720 }, },
716 "moment": {
717 "version": "2.28.0",
718 "resolved": "https://registry.npmjs.org/moment/-/moment-2.28.0.tgz",
719 "integrity": "sha512-Z5KOjYmnHyd/ukynmFd/WwyXHd7L4J9vTI/nn5Ap9AVUgaAE15VvQ9MOGmJJygEUklupqIrFnor/tjTwRU+tQw=="
720 },
721 721 "ms": { "ms": {
722 722 "version": "2.1.2", "version": "2.1.2",
723 723 "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
File package.json changed (mode: 100644) (index ba8768a..129b1f2)
4 4 "description": "Getting Harris County Evictions information", "description": "Getting Harris County Evictions information",
5 5 "main": "main.js", "main": "main.js",
6 6 "dependencies": { "dependencies": {
7 "dayjs": "^1.8.36",
7 8 "domhandler": "^3.0.0", "domhandler": "^3.0.0",
8 "domutils": "^2.2.0",
9 "domutils": "^2.3.0",
9 10 "htmlparser2": "^4.1.0", "htmlparser2": "^4.1.0",
10 "moment": "^2.28.0",
11 11 "mssql": "^6.2.1", "mssql": "^6.2.1",
12 12 "sqlite3": "^5.0.0", "sqlite3": "^5.0.0",
13 13 "tough-cookie": "^4.0.0" "tough-cookie": "^4.0.0"
File updateStatus.js changed (mode: 100644) (index 66cb763..df4f282)
... ... opts.connectCallback = (database) => {
32 32 let promises = []; let promises = [];
33 33 Case.sqlAllCases({ Case.sqlAllCases({
34 34 database: database, database: database,
35 query: "SELECT case_URL FROM cases WHERE case_status IS NULL OR filed_Date IS NULL LIMIT 100",
35 36 // This is the callback used for each case: // This is the callback used for each case:
36 37 rowCallback: caseObj => { rowCallback: caseObj => {
37 38 promises.push( promises.push(
Hints:
Before first commit, do not forget to setup your git environment:
git config --global user.name "your_name_here"
git config --global user.email "your@email_here"

Clone this repository using HTTP(S):
git clone https://rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using ssh (do not forget to upload a key first):
git clone ssh://rocketgit@ssh.rocketgit.com/user/datajams-lbai/datajams-evictions

Clone this repository using git:
git clone git://git.rocketgit.com/user/datajams-lbai/datajams-evictions

You are allowed to anonymously push to this repository.
This means that your pushed commits will automatically be transformed into a merge request:
... clone the repository ...
... make some changes and some commits ...
git push origin main