feat: deduplicate code snippets in tok_read

This commit is contained in:
s3lph 2024-11-29 23:43:50 +01:00
parent 491807dfdb
commit 2705099d30
Signed by: s3lph
GPG key ID: 0AA29A52FB33CFB5

View file

@ -51,10 +51,16 @@ void tok_init(struct tokenizer *t, int (*cb)(const char *tok, uint8_t last)) {
int tok_read(struct tokenizer *t, char *buf, size_t n) {
char *start = buf;
char *end = start;
char *end = NULL, *tmpend = NULL;
while (!t->eot) {
// non-final tokens: terminated by ;
end = strchr(start, ';');
// last token: terminated by .
tmpend = strchr(start, '.');
if (tmpend && (tmpend < end || !end)) {
end = tmpend;
t->eot = 1;
}
if (end) {
if (end > start) {
for (char *c = start; c < end; ++c) {
@ -64,7 +70,7 @@ int tok_read(struct tokenizer *t, char *buf, size_t n) {
}
}
*(t->tok) = 0;
if (t->cb(t->tokbuf, 0)) {
if (t->cb(t->tokbuf, t->eot)) {
return 1;
}
t->tok = t->tokbuf;
@ -72,25 +78,6 @@ int tok_read(struct tokenizer *t, char *buf, size_t n) {
start = end + 1;
continue;
}
// last token: terminated by .
end = strchr(start, '.');
if (end) {
t->eot = 1;
if (end > start) {
for (char *c = start; c < end; ++c) {
if (isprint(*c)) {
*(t->tok++) = *c;
}
}
}
*(t->tok) = 0;
if (t->cb(t->tokbuf, 1)) {
return 1;
}
t->tok = t->tokbuf;
*(t->tok) = 0;
break;
}
// incomplete tokens continued in the next message
end = strchr(start, '?');
if (end) {