From 2a4afad91a72defee1f97f189bba0920bd63fd9e Mon Sep 17 00:00:00 2001 From: scribblemaniac Date: Fri, 15 Oct 2021 22:53:53 -0600 Subject: [PATCH 1/7] Add checkpointing support for passphrases --- main.c | 19 +++++++++++++++++++ worker.c | 2 ++ worker.h | 2 ++ worker_batch_pass.inc.h | 23 ++++++++++++++++++++++- worker_fast_pass.inc.h | 23 ++++++++++++++++++++++- 5 files changed, 67 insertions(+), 2 deletions(-) diff --git a/main.c b/main.c index cec31bb..05bfcb7 100644 --- a/main.c +++ b/main.c @@ -112,6 +112,7 @@ static void printhelp(FILE *out,const char *progname) #ifdef PASSPHRASE "\t-p passphrase - use passphrase to initialize the random seed with\n" "\t-P - same as -p, but takes passphrase from PASSPHRASE environment variable\n" + "\t-c filename - load/save checkpoint of progress to specified file (requires passphrase)\n" #endif ,progname,progname); fflush(out); @@ -393,6 +394,12 @@ int main(int argc,char **argv) setpassphrase(pass); deterministic = 1; } + else if (*arg == 'c') { + if (argc--) + checkpointfile = *argv++; + else + e_additional(); + } #endif // PASSPHRASE else { fprintf(stderr,"unrecognised argument: -%c\n",*arg); @@ -465,6 +472,18 @@ int main(int argc,char **argv) goto done; } + if (checkpointfile) { + // Read current checkpoint position if file exists + FILE *checkout = fopen(checkpointfile, "r"); + if (checkout) { + if(fread(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { + fprintf(stderr,"failed to read checkpoint file\n"); + exit(1); + } + fclose(checkout); + } + } + filters_prepare(); filters_print(); diff --git a/worker.c b/worker.c index f0aaf62..475d2a3 100644 --- a/worker.c +++ b/worker.c @@ -63,6 +63,8 @@ void worker_init(void) pthread_mutex_t determseed_mutex; u8 determseed[SEED_LEN]; +u8 checkpoint[SEED_LEN]; +const char *checkpointfile = 0; #endif diff --git a/worker.h b/worker.h index 7f50f6f..d20d037 100644 --- a/worker.h +++ b/worker.h @@ -33,6 +33,8 @@ VEC_STRUCT(statsvec,struct statstruct); #ifdef PASSPHRASE extern pthread_mutex_t determseed_mutex; extern u8 determseed[SEED_LEN]; +extern u8 checkpoint[SEED_LEN]; +extern const char *checkpointfile; #endif extern void worker_init(void); diff --git a/worker_batch_pass.inc.h b/worker_batch_pass.inc.h index 3676eec..ecdcb45 100644 --- a/worker_batch_pass.inc.h +++ b/worker_batch_pass.inc.h @@ -36,18 +36,39 @@ void *worker_batch_pass(void *task) sname = makesname(); + // load checkpoint + pthread_mutex_lock(&determseed_mutex); + for (int i = 0; i < SEED_LEN; i++) + determseed[i] += checkpoint[i]; + pthread_mutex_unlock(&determseed_mutex); + initseed: #ifdef STATISTICS ++st->numrestart.v; #endif pthread_mutex_lock(&determseed_mutex); - for (int i = 0; i < SEED_LEN; i++) + for (int i = 0; i < SEED_LEN; i++) { + ++checkpoint[i]; if (++determseed[i]) break; + } memcpy(seed, determseed, SEED_LEN); pthread_mutex_unlock(&determseed_mutex); + if (checkpointfile) { + FILE *checkout = fopen(checkpointfile, "w"); + if (!checkout) { + fprintf(stderr,"cannot open checkpoint file for writing\n"); + exit(1); + } + if(fwrite(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { + fprintf(stderr,"cannot write to checkpoint file\n"); + exit(1); + } + fclose(checkout); + } + ed25519_seckey_expand(sk,seed); ge_scalarmult_base(&ge_public,sk); diff --git a/worker_fast_pass.inc.h b/worker_fast_pass.inc.h index 2d482b3..3f281b5 100644 --- a/worker_fast_pass.inc.h +++ b/worker_fast_pass.inc.h @@ -33,18 +33,39 @@ void *worker_fast_pass(void *task) sname = makesname(); + // load checkpoint + pthread_mutex_lock(&determseed_mutex); + for (int i = 0; i < SEED_LEN; i++) + determseed[i] += checkpoint[i]; + pthread_mutex_unlock(&determseed_mutex); + initseed: #ifdef STATISTICS ++st->numrestart.v; #endif pthread_mutex_lock(&determseed_mutex); - for (int i = 0; i < SEED_LEN; i++) + for (int i = 0; i < SEED_LEN; i++) { + ++checkpoint[i]; if (++determseed[i]) break; + } memcpy(seed, determseed, SEED_LEN); pthread_mutex_unlock(&determseed_mutex); + if (checkpointfile) { + FILE *checkout = fopen(checkpointfile, "w"); + if (!checkout) { + fprintf(stderr,"cannot open checkpoint file for writing\n"); + exit(1); + } + if(fwrite(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { + fprintf(stderr,"cannot write to checkpoint file\n"); + exit(1); + } + fclose(checkout); + } + ed25519_seckey_expand(sk,seed); ge_scalarmult_base(&ge_public,sk); From c9396de8b2bd65084c82e3bde52b041f18e6ff15 Mon Sep 17 00:00:00 2001 From: scribblemaniac Date: Fri, 5 Nov 2021 17:27:14 -0600 Subject: [PATCH 2/7] Use long argument for checkpoint --- main.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/main.c b/main.c index 05bfcb7..92031d1 100644 --- a/main.c +++ b/main.c @@ -112,7 +112,7 @@ static void printhelp(FILE *out,const char *progname) #ifdef PASSPHRASE "\t-p passphrase - use passphrase to initialize the random seed with\n" "\t-P - same as -p, but takes passphrase from PASSPHRASE environment variable\n" - "\t-c filename - load/save checkpoint of progress to specified file (requires passphrase)\n" + "\t--checkpoint filename - load/save checkpoint of progress to specified file (requires passphrase)\n" #endif ,progname,progname); fflush(out); @@ -249,6 +249,14 @@ int main(int argc,char **argv) } else if (!strcmp(arg,"rawyaml")) yamlraw = 1; +#ifdef PASSPHRASE + else if (!strcmp(arg,"checkpoint")) { + if (argc--) + checkpointfile = *argv++; + else + e_additional(); + } +#endif // PASSPHRASE else { fprintf(stderr,"unrecognised argument: --%s\n",arg); exit(1); @@ -394,12 +402,6 @@ int main(int argc,char **argv) setpassphrase(pass); deterministic = 1; } - else if (*arg == 'c') { - if (argc--) - checkpointfile = *argv++; - else - e_additional(); - } #endif // PASSPHRASE else { fprintf(stderr,"unrecognised argument: -%c\n",*arg); From 3706518f76ea35478e304df094693ade65544734 Mon Sep 17 00:00:00 2001 From: scribblemaniac Date: Fri, 5 Nov 2021 19:26:21 -0600 Subject: [PATCH 3/7] Move checkpoint saving to main thread Checkpoints will now be saved every 5 minutes and when the program ends. --- main.c | 79 +++++++++++++++++++++++++++++++++-------- worker.c | 2 -- worker.h | 2 -- worker_batch_pass.inc.h | 23 +----------- worker_fast_pass.inc.h | 23 +----------- 5 files changed, 67 insertions(+), 62 deletions(-) diff --git a/main.c b/main.c index 92031d1..85e637c 100644 --- a/main.c +++ b/main.c @@ -58,6 +58,11 @@ size_t printlen; // precalculated, related to printstartpos pthread_mutex_t fout_mutex; FILE *fout; +#ifdef PASSPHRASE +u8 orig_determseed[SEED_LEN]; +const char *checkpointfile = 0; +#endif + static void termhandler(int sig) { switch (sig) { @@ -174,6 +179,35 @@ static void setpassphrase(const char *pass) } fprintf(stderr," done.\n"); } + +static void savecheckpoint(void) +{ + if (checkpointfile) { + // Open checkpoint file + FILE *checkout = fopen(checkpointfile, "w"); + if (!checkout) { + fprintf(stderr,"cannot open checkpoint file for writing\n"); + exit(1); + } + + // Calculate checkpoint as the difference between original seed and the current seed + u8 checkpoint[SEED_LEN]; + bool carry = 0; + pthread_mutex_lock(&determseed_mutex); + for (int i = 0; i < SEED_LEN; i++) { + checkpoint[i] = determseed[i] - orig_determseed[i] - carry; + carry = checkpoint[i] > determseed[i]; + } + pthread_mutex_unlock(&determseed_mutex); + + // Write checkpoint file + if(fwrite(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { + fprintf(stderr,"cannot write to checkpoint file\n"); + exit(1); + } + fclose(checkout); + } +} #endif VEC_STRUCT(threadvec, pthread_t); @@ -474,18 +508,6 @@ int main(int argc,char **argv) goto done; } - if (checkpointfile) { - // Read current checkpoint position if file exists - FILE *checkout = fopen(checkpointfile, "r"); - if (checkout) { - if(fread(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { - fprintf(stderr,"failed to read checkpoint file\n"); - exit(1); - } - fclose(checkout); - } - } - filters_prepare(); filters_print(); @@ -521,8 +543,27 @@ int main(int argc,char **argv) numthreads,numthreads == 1 ? "thread" : "threads"); #ifdef PASSPHRASE - if (!quietflag && deterministic && numneedgenerate != 1) - fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n"); + memcpy(orig_determseed, determseed, sizeof(determseed)); + if (deterministic) { + if (!quietflag && numneedgenerate != 1) + fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n"); + if (checkpointfile) { + // Read current checkpoint position if file exists + FILE *checkout = fopen(checkpointfile, "r"); + if (checkout) { + u8 checkpoint[SEED_LEN]; + if(fread(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { + fprintf(stderr,"failed to read checkpoint file\n"); + exit(1); + } + fclose(checkout); + + // Apply checkpoint to determseed + for (int i = 0; i < SEED_LEN; i++) + determseed[i] += checkpoint[i]; + } + } + } #endif signal(SIGTERM,termhandler); @@ -605,11 +646,17 @@ int main(int argc,char **argv) struct timespec ts; memset(&ts,0,sizeof(ts)); ts.tv_nsec = 100000000; + u16 loopcounter = 0; while (!endwork) { if (numneedgenerate && keysgenerated >= numneedgenerate) { endwork = 1; break; } + loopcounter++; + if (loopcounter >= 3000) { // Save checkpoint every 5 minutes + savecheckpoint(); + loopcounter = 0; + } nanosleep(&ts,0); #ifdef STATISTICS @@ -675,6 +722,10 @@ int main(int argc,char **argv) #endif } +#ifdef PASSPHRASE + savecheckpoint(); +#endif + if (!quietflag) fprintf(stderr,"waiting for threads to finish..."); for (size_t i = 0;i < VEC_LENGTH(threads);++i) diff --git a/worker.c b/worker.c index 475d2a3..f0aaf62 100644 --- a/worker.c +++ b/worker.c @@ -63,8 +63,6 @@ void worker_init(void) pthread_mutex_t determseed_mutex; u8 determseed[SEED_LEN]; -u8 checkpoint[SEED_LEN]; -const char *checkpointfile = 0; #endif diff --git a/worker.h b/worker.h index d20d037..7f50f6f 100644 --- a/worker.h +++ b/worker.h @@ -33,8 +33,6 @@ VEC_STRUCT(statsvec,struct statstruct); #ifdef PASSPHRASE extern pthread_mutex_t determseed_mutex; extern u8 determseed[SEED_LEN]; -extern u8 checkpoint[SEED_LEN]; -extern const char *checkpointfile; #endif extern void worker_init(void); diff --git a/worker_batch_pass.inc.h b/worker_batch_pass.inc.h index ecdcb45..3676eec 100644 --- a/worker_batch_pass.inc.h +++ b/worker_batch_pass.inc.h @@ -36,39 +36,18 @@ void *worker_batch_pass(void *task) sname = makesname(); - // load checkpoint - pthread_mutex_lock(&determseed_mutex); - for (int i = 0; i < SEED_LEN; i++) - determseed[i] += checkpoint[i]; - pthread_mutex_unlock(&determseed_mutex); - initseed: #ifdef STATISTICS ++st->numrestart.v; #endif pthread_mutex_lock(&determseed_mutex); - for (int i = 0; i < SEED_LEN; i++) { - ++checkpoint[i]; + for (int i = 0; i < SEED_LEN; i++) if (++determseed[i]) break; - } memcpy(seed, determseed, SEED_LEN); pthread_mutex_unlock(&determseed_mutex); - if (checkpointfile) { - FILE *checkout = fopen(checkpointfile, "w"); - if (!checkout) { - fprintf(stderr,"cannot open checkpoint file for writing\n"); - exit(1); - } - if(fwrite(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { - fprintf(stderr,"cannot write to checkpoint file\n"); - exit(1); - } - fclose(checkout); - } - ed25519_seckey_expand(sk,seed); ge_scalarmult_base(&ge_public,sk); diff --git a/worker_fast_pass.inc.h b/worker_fast_pass.inc.h index 3f281b5..2d482b3 100644 --- a/worker_fast_pass.inc.h +++ b/worker_fast_pass.inc.h @@ -33,39 +33,18 @@ void *worker_fast_pass(void *task) sname = makesname(); - // load checkpoint - pthread_mutex_lock(&determseed_mutex); - for (int i = 0; i < SEED_LEN; i++) - determseed[i] += checkpoint[i]; - pthread_mutex_unlock(&determseed_mutex); - initseed: #ifdef STATISTICS ++st->numrestart.v; #endif pthread_mutex_lock(&determseed_mutex); - for (int i = 0; i < SEED_LEN; i++) { - ++checkpoint[i]; + for (int i = 0; i < SEED_LEN; i++) if (++determseed[i]) break; - } memcpy(seed, determseed, SEED_LEN); pthread_mutex_unlock(&determseed_mutex); - if (checkpointfile) { - FILE *checkout = fopen(checkpointfile, "w"); - if (!checkout) { - fprintf(stderr,"cannot open checkpoint file for writing\n"); - exit(1); - } - if(fwrite(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { - fprintf(stderr,"cannot write to checkpoint file\n"); - exit(1); - } - fclose(checkout); - } - ed25519_seckey_expand(sk,seed); ge_scalarmult_base(&ge_public,sk); From 5b4074a47eaa3e7d22bcabb1c6dfd2f61af551c2 Mon Sep 17 00:00:00 2001 From: cathugger Date: Wed, 8 Dec 2021 20:14:20 +0000 Subject: [PATCH 4/7] make checkpoint stuff actually proper --- ioutil.c | 150 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- ioutil.h | 1 + main.c | 114 ++++++++++++++++++++++++++++-------------- 3 files changed, 225 insertions(+), 40 deletions(-) diff --git a/ioutil.c b/ioutil.c index 4e6aa55..301db77 100644 --- a/ioutil.c +++ b/ioutil.c @@ -2,6 +2,8 @@ #include #include "types.h" #include "ioutil.h" +#include "vec.h" +#include #ifndef _WIN32 @@ -31,7 +33,7 @@ FH createfile(const char *path,int secret) int fd; do { fd = open(path,O_WRONLY | O_CREAT | O_TRUNC,secret ? 0600 : 0666); - if (fd == -1) { + if (fd < 0) { if (errno == EINTR) continue; return -1; @@ -45,7 +47,7 @@ int closefile(FH fd) int cret; do { cret = close(fd); - if (cret == -1) { + if (cret < 0) { if (errno == EINTR) continue; return -1; @@ -59,6 +61,107 @@ int createdir(const char *path,int secret) return mkdir(path,secret ? 0700 : 0777); } +int syncwrite(const char *filename,int secret,const u8 *data,size_t datalen) +{ + //fprintf(stderr,"filename = %s\n",filename); + + VEC_STRUCT(,char) tmpname; + size_t fnlen = strlen(filename); + VEC_INIT(tmpname); + VEC_ADDN(tmpname,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */); + memcpy(&VEC_BUF(tmpname,0),filename,fnlen); + strcpy(&VEC_BUF(tmpname,fnlen),".tmp"); + const char *tmpnamestr = &VEC_BUF(tmpname,0); + + //fprintf(stderr,"tmpnamestr = %s\n",tmpnamestr); + + FH f = createfile(tmpnamestr,secret); + if (f == FH_invalid) + return -1; + + if (writeall(f,data,datalen) < 0) { + closefile(f); + remove(tmpnamestr); + return -1; + } + + int sret; + do { + sret = fsync(f); + if (sret < 0) { + if (errno == EINTR) + continue; + + closefile(f); + remove(tmpnamestr); + return -1; + } + } while (0); + + if (closefile(f) < 0) { + remove(tmpnamestr); + return -1; + } + + if (rename(tmpnamestr,filename) < 0) { + remove(tmpnamestr); + return -1; + } + + VEC_STRUCT(,char) dirname; + const char *dirnamestr; + + for (ssize_t x = ((ssize_t)fnlen) - 1;x >= 0;--x) { + if (filename[x] == '/') { + if (x) + --x; + ++x; + VEC_INIT(dirname); + VEC_ADDN(dirname,x + 1); + memcpy(&VEC_BUF(dirname,0),filename,x); + VEC_BUF(dirname,x) = '\0'; + dirnamestr = &VEC_BUF(dirname,0); + goto foundslash; + } + } + /* not found slash, fall back to "." */ + dirnamestr = "."; + +foundslash: + //fprintf(stderr,"dirnamestr = %s\n",dirnamestr); + ; + + int dirf; + do { + dirf = open(dirnamestr,O_RDONLY); + if (dirf < 0) { + if (errno == EINTR) + continue; + + // failed for non-eintr reasons + goto skipdsync; // don't really care enough + } + } while (0); + + do { + sret = fsync(dirf); + if (sret < 0) { + if (errno == EINTR) + continue; + + // failed for non-eintr reasons + break; // don't care + } + } while (0); + + (void) closefile(dirf); // don't care + +skipdsync: + + return 0; +} + + #else int writeall(FH fd,const u8 *data,size_t len) @@ -99,6 +202,49 @@ int createdir(const char *path,int secret) return CreateDirectoryA(path,0) ? 0 : -1; } + + +int syncwrite(const char *filename,int secret,const char *data,size_t datalen) +{ + VEC_STRUCT(,char) tmpname; + size_t fnlen = strlen(filename); + VEC_INIT(tmpname); + VEC_ADDN(tmpname,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */); + memcpy(&VEC_BUF(tmpname,0),filename,fnlen); + strcpy(&VEC_BUF(tmpname,fnlen),".tmp"); + const char *tmpnamestr = &VEC_BUF(tmpname,0); + + FH f = createfile(tmpnamestr,secret) + if (f == FH_invalid) + return -1; + + if (writeall(f,data,datalen) < 0) { + closefile(f); + remove(tmpnamestr); + return -1; + } + + if (FlushFileBuffers(f) == 0) { + closefile(f); + remove(tmpnamestr); + return -1; + } + + if (closefile(f) < 0) { + remove(tmpnamestr); + return -1; + } + + if (MoveFileA(tmpnamestr,filename) == 0) { + remove(tmpnamestr); + return -1; + } + + // can't fsync parent dir on windows so just end here + + return 0; +} + #endif int writetofile(const char *path,const u8 *data,size_t len,int secret) diff --git a/ioutil.h b/ioutil.h index c7a1dab..5244508 100644 --- a/ioutil.h +++ b/ioutil.h @@ -18,3 +18,4 @@ int closefile(FH fd); int writeall(FH,const u8 *data,size_t len); int writetofile(const char *path,const u8 *data,size_t len,int secret); int createdir(const char *path,int secret); +int syncwrite(const char *filename,int secret,const u8 *data,size_t datalen); diff --git a/main.c b/main.c index 85e637c..30c630e 100644 --- a/main.c +++ b/main.c @@ -29,6 +29,8 @@ #include "worker.h" +#include "likely.h" + #ifndef _WIN32 #define FSZ "%zu" #else @@ -182,35 +184,55 @@ static void setpassphrase(const char *pass) static void savecheckpoint(void) { - if (checkpointfile) { - // Open checkpoint file - FILE *checkout = fopen(checkpointfile, "w"); - if (!checkout) { - fprintf(stderr,"cannot open checkpoint file for writing\n"); - exit(1); - } - - // Calculate checkpoint as the difference between original seed and the current seed - u8 checkpoint[SEED_LEN]; - bool carry = 0; - pthread_mutex_lock(&determseed_mutex); - for (int i = 0; i < SEED_LEN; i++) { - checkpoint[i] = determseed[i] - orig_determseed[i] - carry; - carry = checkpoint[i] > determseed[i]; - } - pthread_mutex_unlock(&determseed_mutex); - - // Write checkpoint file - if(fwrite(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { - fprintf(stderr,"cannot write to checkpoint file\n"); - exit(1); - } - fclose(checkout); + u8 checkpoint[SEED_LEN]; + bool carry = 0; + pthread_mutex_lock(&determseed_mutex); + for (int i = 0; i < SEED_LEN; i++) { + checkpoint[i] = determseed[i] - orig_determseed[i] - carry; + carry = checkpoint[i] > determseed[i]; } + pthread_mutex_unlock(&determseed_mutex); + + if (syncwrite(checkpointfile,1,checkpoint,SEED_LEN) < 0) { + pthread_mutex_lock(&fout_mutex); + fprintf(stderr,"ERROR: could not save checkpoint\n"); + pthread_mutex_unlock(&fout_mutex); + } +} + +static volatile int checkpointer_endwork = 0; + +static void *checkpointworker(void *arg) +{ + (void) arg; + + struct timespec ts; + memset(&ts,0,sizeof(ts)); + ts.tv_nsec = 100000000; + + struct timespec nowtime; + u64 ilasttime,inowtime; + clock_gettime(CLOCK_MONOTONIC,&nowtime); + ilasttime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000); + + while (!unlikely(checkpointer_endwork)) { + + clock_gettime(CLOCK_MONOTONIC,&nowtime); + inowtime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000); + + if (inowtime - ilasttime >= 300 * 1000000 /* 5 minutes */) { + savecheckpoint(); + ilasttime = inowtime; + } + } + + savecheckpoint(); + + return 0; } #endif -VEC_STRUCT(threadvec, pthread_t); +VEC_STRUCT(threadvec,pthread_t); #include "filters_inc.inc.h" #include "filters_main.inc.h" @@ -458,6 +480,11 @@ int main(int argc,char **argv) exit(1); } + if (checkpointfile && !deterministic) { + fprintf(stderr,"--checkpoint requires passphrase\n"); + exit(1); + } + if (outfile) { fout = fopen(outfile,!outfileoverwrite ? "a" : "w"); if (!fout) { @@ -543,16 +570,16 @@ int main(int argc,char **argv) numthreads,numthreads == 1 ? "thread" : "threads"); #ifdef PASSPHRASE - memcpy(orig_determseed, determseed, sizeof(determseed)); if (deterministic) { + memcpy(orig_determseed,determseed,sizeof(determseed)); if (!quietflag && numneedgenerate != 1) fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n"); if (checkpointfile) { // Read current checkpoint position if file exists - FILE *checkout = fopen(checkpointfile, "r"); + FILE *checkout = fopen(checkpointfile,"r"); if (checkout) { u8 checkpoint[SEED_LEN]; - if(fread(checkpoint, 1, SEED_LEN, checkout) != SEED_LEN) { + if(fread(checkpoint,1,SEED_LEN,checkout) != SEED_LEN) { fprintf(stderr,"failed to read checkpoint file\n"); exit(1); } @@ -634,6 +661,18 @@ int main(int argc,char **argv) perror("pthread_attr_destroy"); } +#if PASSPHRASE + pthread_t checkpoint_thread; + + if (checkpointfile) { + tret = pthread_create(&checkpoint_thread,NULL,checkpointworker,NULL); + if (tret) { + fprintf(stderr,"error while making checkpoint thread: %s\n",strerror(tret)); + exit(1); + } + } +#endif + #ifdef STATISTICS struct timespec nowtime; u64 istarttime,inowtime,ireporttime = 0,elapsedoffset = 0; @@ -643,20 +682,15 @@ int main(int argc,char **argv) } istarttime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000); #endif + struct timespec ts; memset(&ts,0,sizeof(ts)); ts.tv_nsec = 100000000; - u16 loopcounter = 0; while (!endwork) { if (numneedgenerate && keysgenerated >= numneedgenerate) { endwork = 1; break; } - loopcounter++; - if (loopcounter >= 3000) { // Save checkpoint every 5 minutes - savecheckpoint(); - loopcounter = 0; - } nanosleep(&ts,0); #ifdef STATISTICS @@ -722,14 +756,18 @@ int main(int argc,char **argv) #endif } -#ifdef PASSPHRASE - savecheckpoint(); -#endif - if (!quietflag) fprintf(stderr,"waiting for threads to finish..."); + for (size_t i = 0;i < VEC_LENGTH(threads);++i) pthread_join(VEC_BUF(threads,i),0); +#ifdef PASSPHRASE + if (checkpointfile) { + checkpointer_endwork = 1; + pthread_join(checkpoint_thread,0); + } +#endif + if (!quietflag) fprintf(stderr," done.\n"); From 90fe9f35d1d74dd794452e0372cdfc6742fe4dd3 Mon Sep 17 00:00:00 2001 From: cathugger Date: Wed, 8 Dec 2021 20:22:24 +0000 Subject: [PATCH 5/7] some tweaks --- ioutil.c | 3 --- main.c | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ioutil.c b/ioutil.c index 301db77..89fd1c3 100644 --- a/ioutil.c +++ b/ioutil.c @@ -161,7 +161,6 @@ skipdsync: return 0; } - #else int writeall(FH fd,const u8 *data,size_t len) @@ -202,8 +201,6 @@ int createdir(const char *path,int secret) return CreateDirectoryA(path,0) ? 0 : -1; } - - int syncwrite(const char *filename,int secret,const char *data,size_t datalen) { VEC_STRUCT(,char) tmpname; diff --git a/main.c b/main.c index 30c630e..eac7cd6 100644 --- a/main.c +++ b/main.c @@ -220,7 +220,7 @@ static void *checkpointworker(void *arg) clock_gettime(CLOCK_MONOTONIC,&nowtime); inowtime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000); - if (inowtime - ilasttime >= 300 * 1000000 /* 5 minutes */) { + if ((i64)(inowtime - ilasttime) >= 300 * 1000000 /* 5 minutes */) { savecheckpoint(); ilasttime = inowtime; } @@ -718,7 +718,7 @@ int main(int argc,char **argv) VEC_BUF(tstats,i).numrestart += (u64)tdiff; sumrestart += VEC_BUF(tstats,i).numrestart; } - if (reportdelay && (!ireporttime || inowtime - ireporttime >= reportdelay)) { + if (reportdelay && (!ireporttime || (i64)(inowtime - ireporttime) >= (i64)reportdelay)) { if (ireporttime) ireporttime += reportdelay; else From 1679e51e1b94daf35b8f71f8ea96a5ddaf41627e Mon Sep 17 00:00:00 2001 From: cathugger Date: Wed, 8 Dec 2021 20:24:55 +0000 Subject: [PATCH 6/7] move this there --- main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.c b/main.c index eac7cd6..12bd1f3 100644 --- a/main.c +++ b/main.c @@ -571,10 +571,10 @@ int main(int argc,char **argv) #ifdef PASSPHRASE if (deterministic) { - memcpy(orig_determseed,determseed,sizeof(determseed)); if (!quietflag && numneedgenerate != 1) fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n"); if (checkpointfile) { + memcpy(orig_determseed,determseed,sizeof(determseed)); // Read current checkpoint position if file exists FILE *checkout = fopen(checkpointfile,"r"); if (checkout) { From 67868f41264e0bb43d7572e581e29626ebaf696a Mon Sep 17 00:00:00 2001 From: cathugger Date: Fri, 10 Dec 2021 13:27:01 +0200 Subject: [PATCH 7/7] cleanup, free memory --- ioutil.c | 139 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 85 insertions(+), 54 deletions(-) diff --git a/ioutil.c b/ioutil.c index 89fd1c3..cc430df 100644 --- a/ioutil.c +++ b/ioutil.c @@ -1,5 +1,6 @@ #include #include +#include #include "types.h" #include "ioutil.h" #include "vec.h" @@ -61,28 +62,14 @@ int createdir(const char *path,int secret) return mkdir(path,secret ? 0700 : 0777); } -int syncwrite(const char *filename,int secret,const u8 *data,size_t datalen) +static int syncwritefile(const char *filename,const char *tmpname,int secret,const u8 *data,size_t datalen) { - //fprintf(stderr,"filename = %s\n",filename); - - VEC_STRUCT(,char) tmpname; - size_t fnlen = strlen(filename); - VEC_INIT(tmpname); - VEC_ADDN(tmpname,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */); - memcpy(&VEC_BUF(tmpname,0),filename,fnlen); - strcpy(&VEC_BUF(tmpname,fnlen),".tmp"); - const char *tmpnamestr = &VEC_BUF(tmpname,0); - - //fprintf(stderr,"tmpnamestr = %s\n",tmpnamestr); - - FH f = createfile(tmpnamestr,secret); + FH f = createfile(tmpname,secret); if (f == FH_invalid) return -1; if (writeall(f,data,datalen) < 0) { - closefile(f); - remove(tmpnamestr); - return -1; + goto failclose; } int sret; @@ -92,48 +79,76 @@ int syncwrite(const char *filename,int secret,const u8 *data,size_t datalen) if (errno == EINTR) continue; - closefile(f); - remove(tmpnamestr); - return -1; + goto failclose; } } while (0); if (closefile(f) < 0) { - remove(tmpnamestr); - return -1; + goto failrm; } - if (rename(tmpnamestr,filename) < 0) { - remove(tmpnamestr); - return -1; + if (rename(tmpname,filename) < 0) { + goto failrm; } - VEC_STRUCT(,char) dirname; - const char *dirnamestr; + return 0; + +failclose: + (void) closefile(f); +failrm: + remove(tmpname); + + return -1; +} + +int syncwrite(const char *filename,int secret,const u8 *data,size_t datalen) +{ + //fprintf(stderr,"filename = %s\n",filename); + + size_t fnlen = strlen(filename); + + VEC_STRUCT(,char) tmpnamebuf; + VEC_INIT(tmpnamebuf); + VEC_ADDN(tmpnamebuf,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */); + memcpy(&VEC_BUF(tmpnamebuf,0),filename,fnlen); + strcpy(&VEC_BUF(tmpnamebuf,fnlen),".tmp"); + const char *tmpname = &VEC_BUF(tmpnamebuf,0); + + //fprintf(stderr,"tmpname = %s\n",tmpname); + + int r = syncwritefile(filename,tmpname,secret,data,datalen); + + VEC_FREE(tmpnamebuf); + + if (r < 0) + return r; + + VEC_STRUCT(,char) dirnamebuf; + VEC_INIT(dirnamebuf); + const char *dirname; for (ssize_t x = ((ssize_t)fnlen) - 1;x >= 0;--x) { if (filename[x] == '/') { if (x) --x; ++x; - VEC_INIT(dirname); - VEC_ADDN(dirname,x + 1); - memcpy(&VEC_BUF(dirname,0),filename,x); - VEC_BUF(dirname,x) = '\0'; - dirnamestr = &VEC_BUF(dirname,0); + VEC_ADDN(dirnamebuf,x + 1); + memcpy(&VEC_BUF(dirnamebuf,0),filename,x); + VEC_BUF(dirnamebuf,x) = '\0'; + dirname = &VEC_BUF(dirnamebuf,0); goto foundslash; } } /* not found slash, fall back to "." */ - dirnamestr = "."; + dirname = "."; foundslash: - //fprintf(stderr,"dirnamestr = %s\n",dirnamestr); + //fprintf(stderr,"dirname = %s\n",dirname); ; int dirf; do { - dirf = open(dirnamestr,O_RDONLY); + dirf = open(dirname,O_RDONLY); if (dirf < 0) { if (errno == EINTR) continue; @@ -143,6 +158,7 @@ foundslash: } } while (0); + int sret; do { sret = fsync(dirf); if (sret < 0) { @@ -157,6 +173,7 @@ foundslash: (void) closefile(dirf); // don't care skipdsync: + VEC_FREE(dirnamebuf); return 0; } @@ -201,42 +218,56 @@ int createdir(const char *path,int secret) return CreateDirectoryA(path,0) ? 0 : -1; } -int syncwrite(const char *filename,int secret,const char *data,size_t datalen) +static int syncwritefile(const char *filename,const char *tmpname,int secret,const char *data,size_t datalen) { - VEC_STRUCT(,char) tmpname; - size_t fnlen = strlen(filename); - VEC_INIT(tmpname); - VEC_ADDN(tmpname,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */); - memcpy(&VEC_BUF(tmpname,0),filename,fnlen); - strcpy(&VEC_BUF(tmpname,fnlen),".tmp"); - const char *tmpnamestr = &VEC_BUF(tmpname,0); - FH f = createfile(tmpnamestr,secret) if (f == FH_invalid) return -1; if (writeall(f,data,datalen) < 0) { - closefile(f); - remove(tmpnamestr); - return -1; + goto failclose; } if (FlushFileBuffers(f) == 0) { - closefile(f); - remove(tmpnamestr); - return -1; + goto failclose; } if (closefile(f) < 0) { - remove(tmpnamestr); - return -1; + goto failrm; } if (MoveFileA(tmpnamestr,filename) == 0) { - remove(tmpnamestr); - return -1; + goto failrm; } + return 0; + +failclose: + (void) closefile(f); +failrm: + remove(tmpnamestr); + + return -1; +} + +int syncwrite(const char *filename,int secret,const char *data,size_t datalen) +{ + size_t fnlen = strlen(filename); + + VEC_STRUCT(,char) tmpnamebuf; + VEC_INIT(tmpnamebuf); + VEC_ADDN(tmpnamebuf,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */); + memcpy(&VEC_BUF(tmpnamebuf,0),filename,fnlen); + strcpy(&VEC_BUF(tmpnamebuf,fnlen),".tmp"); + const char *tmpname = &VEC_BUF(tmpnamebuf,0); + + int r = syncwritefile(filename,tmpname,secret,data,datalen); + + VEC_FREE(tmpnamebuf); + + if (r < 0) + return r; + // can't fsync parent dir on windows so just end here return 0;