catalinux / dupdump (public) (License: GPLv3) (since 2016-03-01) (hash sha1)
Find duplicated files and directories.
Clone URLs: https://rocketgit.com/user/catalinux/dupdump ssh://rocketgit@ssh.rocketgit.com/user/catalinux/dupdump git://git.rocketgit.com/user/catalinux/dupdump
main master
List of commits:
Subject Hash Author Date (UTC)
Changed scope for some variables 8e195c8a9e8972831c9e25ce36e662cd808f4953 Catalin(ux) M. BOIE 2016-03-19 11:49:15
Dump also the size of the dir/file 1c4995ce058d04f2e7ea61d370c35df220226a6b Catalin(ux) M. BOIE 2015-10-18 15:07:43
WIP 1ccb2423931f1fe25232181b9d056cdb85aefd79 Catalin(ux) M. BOIE 2015-10-04 09:45:00
WIP 6a07b5768407ffb1565471546fd9b1e1f857af48 Catalin(ux) M. BOIE 2015-10-02 22:49:33
WIP f1a3e07fc8a2b9779071537baa7457c91a7a4381 Catalin(ux) M. BOIE 2015-10-01 17:55:26
More compiler flags, several fixes as result 9b63341f8a3052dabe9a402b7c70e33a99633149 Catalin(ux) M. BOIE 2015-01-05 17:59:41
WIP b303e4a78b96847ac107e6b6b905b41629401a75 Catalin(ux) M. BOIE 2014-06-22 20:43:08
Bump version to 0.3 d8ec07e06ac315f40cf7b03e0f5c0392bcd56059 Catalin(ux) M. BOIE 2014-06-22 20:04:09
Updated TODO to reflect reality. 38351ac1d1167674339839c4518724f1ebd7bc89 Catalin(ux) M. BOIE 2014-06-22 20:03:34
Take care of mangled file name between two dirs. 65eb5f1841aae521c401ab52298041776159a35d Catalin(ux) M. BOIE 2014-06-21 12:14:36
duilder updates. Exclude Makefile when making tar.gz accfc9df80b5889b3e1f1183e5f60d70850b5671 Catalin(ux) M. BOIE 2014-06-21 06:09:12
Bump version to 0.2 078df2ea0759e4240ef7deef35744f7077957129 Catalin(ux) M. BOIE 2014-06-19 17:09:39
Fixed a case when some files were not dumped at all 0c69ce3cb20aae442da94cff9606936461177e1d Catalin(ux) M. BOIE 2014-06-19 17:05:16
First version that passes all tests. 9627508618bc2c783da838c2332e6896e3004c99 Catalin(ux) M. BOIE 2014-06-18 18:02:32
Fix a little problem with the man page. 1305ac52823f0206b98181c6435049f339be9c53 Catalin(ux) M. BOIE 2013-02-18 19:55:43
Lots of stuff 9c43842ac36feff6b29cb20a95ad4510a23bf472 Catalin(ux) M. BOIE 2012-07-20 15:43:51
Cosmetic + man a53df11bfc30c152c9f61fdb1bcc69dc6ec20765 Catalin(ux) M. BOIE 2012-06-24 12:11:17
First working version\! 27bd1bf47c9fb707760d84ea3cf4241083fa283d Catalin(ux) M. BOIE 2012-06-22 21:10:30
Several fixes 2909b1ba2e99929e775ddfea5f4894c50694a638 Catalin(ux) M. BOIE 2012-06-19 13:08:50
First version 3d7935d9b8a91694fe8213998ce4d3910348d6ef Catalin(ux) M. BOIE 2012-05-06 19:40:40
Commit 8e195c8a9e8972831c9e25ce36e662cd808f4953 - Changed scope for some variables
Author: Catalin(ux) M. BOIE
Author date (UTC): 2016-03-19 11:49
Committer name: Catalin(ux) M. BOIE
Committer date (UTC): 2016-03-19 13:33
Parent(s): 1c4995ce058d04f2e7ea61d370c35df220226a6b
Signing key:
Tree: c96c12cd212dec52d45d3654235b3cf1a67efa17
File Lines added Lines deleted
.gitignore 1 0
Makefile.in 1 0
dupdump.c 3 1
store.c 19 9
File .gitignore changed (mode: 100644) (index 3563188..9e50f3b)
... ... my1.sh
10 10 *.diff *.diff
11 11 Changelog-last Changelog-last
12 12 *.log *.log
13 cppcheck.out
File Makefile.in changed (mode: 100644) (index ac4e696..9d84b67)
... ... install: all
35 35
36 36 test: all test: all
37 37 make -C tests make -C tests
38 cppcheck --enable=all . 2>cppcheck.out
File dupdump.c changed (mode: 100644) (index 665ce38..692f9be)
... ... int main(int argc, char *argv[])
97 97 int c; int c;
98 98 FILE *out; FILE *out;
99 99 struct timespec start, end; struct timespec start, end;
100 time_t diff;
101 100
102 101 while ((c = getopt_long(argc, argv, "zi:o:vdh", options, &options_index)) != -1) { while ((c = getopt_long(argc, argv, "zi:o:vdh", options, &options_index)) != -1) {
103 102 switch (c) { switch (c) {
 
... ... int main(int argc, char *argv[])
129 128 if (optind >= argc) { if (optind >= argc) {
130 129 usage(); usage();
131 130 fprintf(stderr, "Error: no dirs to scan specified!\n"); fprintf(stderr, "Error: no dirs to scan specified!\n");
131 fclose(out);
132 132 return 1; return 1;
133 133 } }
134 134
 
... ... int main(int argc, char *argv[])
181 181 clock_gettime(CLOCK_MONOTONIC, &end); clock_gettime(CLOCK_MONOTONIC, &end);
182 182
183 183 if (verbose) { if (verbose) {
184 time_t diff;
185
184 186 dump_stats(); dump_stats();
185 187
186 188 diff = end.tv_sec - start.tv_sec; diff = end.tv_sec - start.tv_sec;
File store.c changed (mode: 100644) (index 6861842..a87737e)
... ... static int sha1(const char *file, const size_t len, unsigned char *out)
94 94 int fd; int fd;
95 95 unsigned char buf[65536]; unsigned char buf[65536];
96 96 SHA_CTX c; SHA_CTX c;
97 ssize_t n;
98 97 size_t bytes; size_t bytes;
99 98
100 99 fd = open(file, O_RDONLY); fd = open(file, O_RDONLY);
 
... ... static int sha1(const char *file, const size_t len, unsigned char *out)
104 103 SHA1_Init(&c); SHA1_Init(&c);
105 104 bytes = 0; bytes = 0;
106 105 while (bytes < len) { while (bytes < len) {
106 ssize_t n;
107
107 108 n = read(fd, buf, sizeof(buf)); n = read(fd, buf, sizeof(buf));
108 109 if (n == -1) { if (n == -1) {
109 110 close(fd); close(fd);
 
... ... static void file_mark_up_no_dup_possible(struct file_node *f)
595 596 */ */
596 597 static int compare_file_range(struct file_node *a, struct file_node *b) static int compare_file_range(struct file_node *a, struct file_node *b)
597 598 { {
598 int err;
599 struct file_node *q, *p, *dups, *p_last;
599 struct file_node *q, *p, *dups;
600 600
601 601 if (debug) { if (debug) {
602 602 fprintf(stderr, "compare_file_range:\n"); fprintf(stderr, "compare_file_range:\n");
 
... ... static int compare_file_range(struct file_node *a, struct file_node *b)
616 616
617 617 p = a; p = a;
618 618 while (p != b->hash_next) { while (p != b->hash_next) {
619 struct file_node *p_last;
620
619 621 q = p->hash_next; q = p->hash_next;
620 622 if (q == NULL) if (q == NULL)
621 623 break; break;
622 624
623 625 p_last = p->duplicates; p_last = p->duplicates;
624 626 while (q != b->hash_next) { while (q != b->hash_next) {
627 int err;
628
625 629 /* We do not want to break ->duplicates */ /* We do not want to break ->duplicates */
626 630 if (q->skip_compare == 1) { if (q->skip_compare == 1) {
627 631 q = q->hash_next; q = q->hash_next;
 
... ... static int dir_files_hash(unsigned char *hash, unsigned char *fn,
783 787 struct file_node **u; struct file_node **u;
784 788 unsigned long i, mem; unsigned long i, mem;
785 789 SHA_CTX c, fnh; SHA_CTX c, fnh;
786 char *base_name;
787 790
788 791 if (d->files == NULL) { if (d->files == NULL) {
789 792 memset(hash, 0, SHA_DIGEST_LENGTH); memset(hash, 0, SHA_DIGEST_LENGTH);
 
... ... static int dir_files_hash(unsigned char *hash, unsigned char *fn,
817 820
818 821 i = 0; i = 0;
819 822 while (i < d->no_of_files) { while (i < d->no_of_files) {
823 char *base_name;
824
820 825 SHA1_Update(&c, u[i]->sha1_full, SHA_DIGEST_LENGTH); SHA1_Update(&c, u[i]->sha1_full, SHA_DIGEST_LENGTH);
821 826
822 827 base_name = basename(u[i]->name); base_name = basename(u[i]->name);
 
... ... static long long dir_build_hash(struct dir_node *d)
941 946 unsigned char file_names_sha1[SHA_DIGEST_LENGTH]; unsigned char file_names_sha1[SHA_DIGEST_LENGTH];
942 947 int err; int err;
943 948 long long no_of_possible_dirs = 0; long long no_of_possible_dirs = 0;
944 long long ret;
945 char *base_name;
946 949
947 950 if (debug) if (debug)
948 951 fprintf(stderr, "DEBUG: %s [%s] no_dup_possible=%u\n", fprintf(stderr, "DEBUG: %s [%s] no_dup_possible=%u\n",
 
... ... static long long dir_build_hash(struct dir_node *d)
966 969 /* Compute hashes */ /* Compute hashes */
967 970 subdir = d->subdirs; subdir = d->subdirs;
968 971 while (subdir) { while (subdir) {
972 long long ret;
973
969 974 ret = dir_build_hash(subdir); ret = dir_build_hash(subdir);
970 975 if (ret == -1) if (ret == -1)
971 976 return -1; return -1;
 
... ... static long long dir_build_hash(struct dir_node *d)
988 993 /* At the same time, we build hash of file names */ /* At the same time, we build hash of file names */
989 994 subdir = d->subdirs; subdir = d->subdirs;
990 995 while (subdir) { while (subdir) {
996 char *base_name;
997
991 998 SHA1_Update(&c, subdir->sha1, SHA_DIGEST_LENGTH); SHA1_Update(&c, subdir->sha1, SHA_DIGEST_LENGTH);
992 999
993 1000 base_name = basename(subdir->name); base_name = basename(subdir->name);
 
... ... static unsigned long long dir_find_dups_populate_list(struct dir_node **u,
1055 1062 int dir_find_dups(void) int dir_find_dups(void)
1056 1063 { {
1057 1064 unsigned long long mem, i, j, first, last; unsigned long long mem, i, j, first, last;
1058 struct dir_node *d;
1059 long long err;
1060 1065 char dump[SHA_DIGEST_LENGTH * 2 + 1]; char dump[SHA_DIGEST_LENGTH * 2 + 1];
1061 1066
1062 1067 dir_chain_len = 0; dir_chain_len = 0;
 
... ... int dir_find_dups(void)
1065 1070 fprintf(stderr, "[*] dir_find_dups...\n"); fprintf(stderr, "[*] dir_find_dups...\n");
1066 1071
1067 1072 for (i = 0; i < dir_info_count; i++) { for (i = 0; i < dir_info_count; i++) {
1073 long long err;
1074
1068 1075 err = dir_build_hash(dir_info[i]); err = dir_build_hash(dir_info[i]);
1069 1076 if (err == -1) if (err == -1)
1070 1077 return -1; return -1;
 
... ... int dir_find_dups(void)
1085 1092
1086 1093 j = 0; j = 0;
1087 1094 for (i = 0; i < dir_info_count; i++) { for (i = 0; i < dir_info_count; i++) {
1095 struct dir_node *d;
1096
1088 1097 d = dir_info[i]; d = dir_info[i];
1089 1098 j = dir_find_dups_populate_list(dir_chain, j, d); j = dir_find_dups_populate_list(dir_chain, j, d);
1090 1099
 
... ... __cold static void file_dump_duplicates(struct file_node *f,
1343 1352 __cold void dump_duplicates(const unsigned long long min_size, const unsigned int zero) __cold void dump_duplicates(const unsigned long long min_size, const unsigned int zero)
1344 1353 { {
1345 1354 unsigned int i; unsigned int i;
1346 struct dir_node *d;
1347 1355 struct file_node *f; struct file_node *f;
1348 1356 unsigned int hash; unsigned int hash;
1349 1357
1350 1358 if (debug) if (debug)
1351 1359 fprintf(stderr, "[*] Dump duplicated dirs...\n"); fprintf(stderr, "[*] Dump duplicated dirs...\n");
1352 1360 for (i = 0; i < dir_chain_len; i++) { for (i = 0; i < dir_chain_len; i++) {
1361 struct dir_node *d;
1362
1353 1363 if (debug) if (debug)
1354 1364 fprintf(stderr, "\tdump_duplicates[%u]...\n", i); fprintf(stderr, "\tdump_duplicates[%u]...\n", i);
1355 1365 d = dir_chain[i]; d = dir_chain[i];
Hints:
Before first commit, do not forget to setup your git environment:
git config --global user.name "your_name_here"
git config --global user.email "your@email_here"

Clone this repository using HTTP(S):
git clone https://rocketgit.com/user/catalinux/dupdump

Clone this repository using ssh (do not forget to upload a key first):
git clone ssh://rocketgit@ssh.rocketgit.com/user/catalinux/dupdump

Clone this repository using git:
git clone git://git.rocketgit.com/user/catalinux/dupdump

You are allowed to anonymously push to this repository.
This means that your pushed commits will automatically be transformed into a merge request:
... clone the repository ...
... make some changes and some commits ...
git push origin main