From a6ded4443bfcd5841cf35390a13cc9c2a82bc553 Mon Sep 17 00:00:00 2001 From: Tom Smeding Date: Mon, 27 Jul 2020 19:35:00 +0200 Subject: protocol: Protocol version 2: message replies --- protocol.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/protocol.md b/protocol.md index ac8d86a..8c02017 100644 --- a/protocol.md +++ b/protocol.md @@ -1,4 +1,4 @@ -# tomsg protocol (version 1) +# tomsg protocol (version 2) The underlying transport of the protocol is a plain TCP socket. The individual messages are all line-based; this means that a single message, both @@ -56,18 +56,22 @@ A _response_ from the server will be in one of the following forms: - Response to the client's `history` command; is followed by exactly `` messages of type `history_message` with the same tag. The messages are returned in chronological order (oldest first). -- ` history_message ` +- ` history_message ` - Part of the response to the client's `history` command. Index 0 is the oldest message; index (`` - 1) (from the `history` response) is the newest message in the fragment requested. Timestamps are microseconds since - the UNIX epoch. The message id for each message is globally unique for the - entire server. + the UNIX epoch. The reply message id is -1 if the message is a normal + message, or nonnegative if it is a reply to the referenced message in the + same room. A command is identified by its name, which can be found in the list below. Its arguments are zero or more words, except if for the command in question the last argument is of type `string...` (in which case it may contain spaces and ranges until the end of the line). +Note: all messages on the server have an ID that is globally unique on a single +tomsg server. + - ` version ` - Indicates that the client wishes to speak the protocol with the specified version. For the version of the protocol described in this document, see the @@ -122,14 +126,20 @@ ranges until the end of the line). Also marks the current session as active. - Returns `ok` or `error`. -- ` send ` +- ` send ` - Sends a message to the given room. All room members receive a `_push message` push message on all their sessions (except the session the `send` message was sent from). + If `` is -1, this sends a normal message. Otherwise, + `` must be the id of a message in the given room, in which case + the sent message is a reply to the indicated earlier message. If + `` is not a message in this room, `error` is returned. + Also marks the current session as active. The returned `number` response contains the message id of the message sent. + This number will be non-negative. - Returns `number` or `error`. - ` history ` - Requests the last `` messages in the given room, if the client is a @@ -179,9 +189,11 @@ are listed below. (excluding user X) of all rooms user X is a member of receive a `_push online` push message stating that `` (user X) is now online with `` sessions. -- `_push message ` +- `_push message ` - Sent to all sessions of all members of a room in which a message is - posted, except the session that sent the message. + posted, except the session that sent the message. `` is -1 if + the message is a normal message, and references the replied-to message + otherwise. - `_push invite ` - Sent to all sessions of the invited user after an `invite` command. The `` parameter indicates who invited you to the given room. -- cgit v1.2.3-70-g09d2 From 909ada35ccb617344d244d4e76c9ce85fd2b922b Mon Sep 17 00:00:00 2001 From: Tom Smeding Date: Mon, 27 Jul 2020 20:24:44 +0200 Subject: server: Migrate database to version 2 --- migrate_1_to_2.sh | 39 +++++++++++++++++++++++++++++++++++++++ schema.sql | 6 +++++- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100755 migrate_1_to_2.sh diff --git a/migrate_1_to_2.sh b/migrate_1_to_2.sh new file mode 100755 index 0000000..02269d1 --- /dev/null +++ b/migrate_1_to_2.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +INDB=db.db +OUTDB=db_migrated.db + +# This script does not use ALTER TABLE ADD COLUMN because I'm scared by the +# note in the sqlite documentation stating: +# The ALTER TABLE command works by modifying the SQL text of the schema +# stored in the sqlite_master table. No changes are made to table content. +# Because of this, the execution time of the ALTER TABLE command is +# independent of the amount of data in the table. The ALTER TABLE command +# runs as quickly on a table with 10 million rows as it does on a table +# with 1 row. +# I might be mistaken, but that sounds like a performance issue in later usage. +# Therefore, to be sure, we just copy the entire database here. We don't just +# copy and rename the table inside one database to not unnecessarily double the +# disk footprint of the database. + +inversion="$(sqlite3 "$INDB" 'select version from Meta')" + +if [[ $inversion != "1" ]]; then + echo >&2 "$0 migrates from version 1 to version 2, but the database is at version $inversion." + exit 1 +fi + +if [[ -f "$OUTDB" ]]; then + echo >&2 "Output database $OUTDB already exists; remove before running this script" + exit 1 +fi + +sqlite3 "$OUTDB" Date: Mon, 27 Jul 2020 21:22:42 +0200 Subject: server: Protocol v2, DB v2: message replies --- command.c | 31 +++++++++++++++++++++++-------- command.h | 4 ++-- db.c | 36 ++++++++++++++++++++++++++++-------- db.h | 4 +++- event.c | 3 ++- event.h | 3 ++- 6 files changed, 60 insertions(+), 21 deletions(-) diff --git a/command.c b/command.c index 8fa343d..ba8727e 100644 --- a/command.c +++ b/command.c @@ -232,8 +232,18 @@ static struct cmd_retval cmd_send(struct conn_data *data,const char *tag,const c return RET_OK; } userdata_mark_active(data->userid,data->fd,true); + const char *roomname=args[0]; - const char *message=args[1]; + + i64 replyid; + if(!parse_i64(args[1],&replyid)){ + debug("Connection fd=%d sent an invalid number for 'send': '%s'", + data->fd,args[1]); + return RET_CLOSE(true); + } + + const char *message=args[2]; + if(strlen(message)>MAX_MESSAGE_LEN){ net_send_error(data->fd,tag,"Message too long"); return RET_OK; @@ -247,17 +257,22 @@ static struct cmd_retval cmd_send(struct conn_data *data,const char *tag,const c net_send_error(data->fd,tag,"Not in that room"); return RET_OK; } + if(replyid>=0&&!db_message_exists(roomid,replyid)){ + net_send_error(data->fd,tag,"Replied-to message not found"); + return RET_OK; + } i64 timestamp=make_timestamp(); - i64 msgid=db_create_message(roomid,data->userid,make_timestamp(),message); + i64 msgid=db_create_message(roomid,data->userid,make_timestamp(),replyid,message); bool closed=net_send_number(data->fd,tag,msgid); char *pushbuf=NULL; char *username=db_get_username(data->userid); - i64 pushbuflen=asprintf(&pushbuf,"_push message %s %s %" PRIi64 " %" PRIi64 " %s\n", - roomname,username,timestamp,msgid,message); + i64 pushbuflen=asprintf(&pushbuf, + "_push message %s %s %" PRIi64 " %" PRIi64 " %" PRIi64 " %s\n", + roomname,username,timestamp,msgid,replyid,message); - event_emit_message(timestamp,message,username,roomname); + event_emit_message(timestamp,message,username,roomname,replyid); firebase_send_message(roomname,roomid,username,message); free(username); @@ -317,9 +332,9 @@ static struct cmd_retval history_cmd_helper( for(i64 i=ml.count-1;i>=0;i--){ char *username=db_get_username(ml.list[i].userid); - len=asprintf(&buf,"%s history_message %" PRIi64 " %s %s %" PRIi64 " %" PRIi64 " %s\n", + len=asprintf(&buf,"%s history_message %" PRIi64 " %s %s %" PRIi64 " %" PRIi64 " %" PRIi64 " %s\n", tag,ml.count-1-i,roomname,username,ml.list[i].timestamp, - ml.list[i].msgid,ml.list[i].message); + ml.list[i].msgid,ml.list[i].replyid,ml.list[i].message); free(username); closed=net_send_raw_text(data->fd,buf,len); free(buf); @@ -421,7 +436,7 @@ static const struct cmd_info commands[COMMAND_HASH_MODULUS] = { COMMAND_ENTRY('l', "list_members", 1, false, cmd_list_members), COMMAND_ENTRY('c', "create_room", 0, false, cmd_create_room), COMMAND_ENTRY('i', "invite", 2, false, cmd_invite), - COMMAND_ENTRY('s', "send", 2, true, cmd_send), + COMMAND_ENTRY('s', "send", 3, true, cmd_send), COMMAND_ENTRY('h', "history", 2, false, cmd_history), COMMAND_ENTRY('h', "history_before", 3, false, cmd_history_before), COMMAND_ENTRY('p', "ping", 0, false, cmd_ping), diff --git a/command.h b/command.h index e1226a5..ebfec8c 100644 --- a/command.h +++ b/command.h @@ -4,8 +4,8 @@ #include "conn_data.h" -#define PROTOCOL_VERSION 1 -#define MIN_SUPPORTED_PROTOCOL_VERSION 1 +#define PROTOCOL_VERSION 2 +#define MIN_SUPPORTED_PROTOCOL_VERSION 2 // Returns true if socket should be closed. diff --git a/db.c b/db.c index 0de620d..416b9d3 100644 --- a/db.c +++ b/db.c @@ -10,7 +10,8 @@ #define SQLITE(func,...) do{if(sqlite3_##func(__VA_ARGS__)!=SQLITE_OK){die_sqlite("sqlite3_" #func);}}while(0) -#define DATABASE_VERSION 1 +#define DATABASE_VERSION 2 + #define PASSHASH_OPSLIMIT 3 #define PASSHASH_MEMLIMIT crypto_pwhash_MEMLIMIT_INTERACTIVE @@ -381,16 +382,18 @@ bool db_delete_token(i64 userid,const char *token){ } -i64 db_create_message(i64 roomid,i64 userid,i64 timestamp,const char *message){ +i64 db_create_message(i64 roomid,i64 userid,i64 timestamp,i64 replyid,const char *message){ sqlite3_stmt *stmt; SQLITE(prepare_v2,database, - "insert into Messages (room, user, time, message) " - "values (?, ?, ?, ?)" + "insert into Messages (room, user, time, reply, message) " + "values (?, ?, ?, ?, ?)" ,-1,&stmt,NULL); SQLITE(bind_int64,stmt,1,roomid); SQLITE(bind_int64,stmt,2,userid); SQLITE(bind_int64,stmt,3,timestamp); - SQLITE(bind_blob,stmt,4,message,strlen(message),SQLITE_STATIC); + if(replyid>=0)SQLITE(bind_int64,stmt,4,replyid); + else SQLITE(bind_null,stmt,4); + SQLITE(bind_blob,stmt,5,message,strlen(message),SQLITE_STATIC); if(sqlite3_step(stmt)!=SQLITE_DONE)die_sqlite("sqlite3_step"); SQLITE(finalize,stmt); @@ -407,7 +410,7 @@ struct db_message_list db_get_messages_before(i64 roomid,i64 count,i64 beforeid) sqlite3_stmt *stmt; if(beforeid<0){ SQLITE(prepare_v2,database, - "select id, user, time, message " + "select id, user, time, reply, message " "from Messages " "where room = ? " "order by time desc " @@ -417,7 +420,7 @@ struct db_message_list db_get_messages_before(i64 roomid,i64 count,i64 beforeid) SQLITE(bind_int64,stmt,2,count); } else { SQLITE(prepare_v2,database, - "select id, user, time, message " + "select id, user, time, reply, message " "from Messages " "where room = ? and time < (select time from Messages where id = ?) " "order by time desc " @@ -442,7 +445,12 @@ struct db_message_list db_get_messages_before(i64 roomid,i64 count,i64 beforeid) ml.list[ml.count].roomid=roomid; ml.list[ml.count].userid=sqlite3_column_int64(stmt,1); ml.list[ml.count].timestamp=sqlite3_column_int64(stmt,2); - ml.list[ml.count].message=strdup((const char*)sqlite3_column_text(stmt,3)); + if(sqlite3_column_type(stmt,3)==SQLITE_INTEGER){ + ml.list[ml.count].replyid=sqlite3_column_int64(stmt,3); + } else { + ml.list[ml.count].replyid=-1; // NULL, not a reply + } + ml.list[ml.count].message=strdup((const char*)sqlite3_column_text(stmt,4)); ml.count++; } @@ -452,6 +460,18 @@ struct db_message_list db_get_messages_before(i64 roomid,i64 count,i64 beforeid) return ml; } +bool db_message_exists(i64 roomid, i64 msgid) { + sqlite3_stmt *stmt; + SQLITE(prepare_v2, database, + "select id from Messages where id = ? and room = ?", + -1, &stmt, NULL); + SQLITE(bind_int64, stmt, 1, msgid); + SQLITE(bind_int64, stmt, 2, roomid); + bool success = sqlite3_step(stmt) == SQLITE_ROW; + SQLITE(finalize, stmt); + return success; +} + void db_nullify_name_id(struct db_name_id ni){ if(ni.name)free(ni.name); diff --git a/db.h b/db.h index d1c105b..13df899 100644 --- a/db.h +++ b/db.h @@ -16,6 +16,7 @@ struct db_room_list{ struct db_message{ i64 msgid; i64 roomid,userid,timestamp; + i64 replyid; // message this is a reply to, or -1 if normal message char *message; }; @@ -59,10 +60,11 @@ struct db_strings_list db_user_tokens(i64 userid); bool db_add_token(i64 userid,const char *token); bool db_delete_token(i64 userid,const char *token); -i64 db_create_message(i64 roomid,i64 userid,i64 timestamp,const char *message); // returns msgid +i64 db_create_message(i64 roomid,i64 userid,i64 timestamp,i64 replyid,const char *message); // returns msgid struct db_message_list db_get_messages(i64 roomid,i64 count); // gets latest `count` messages in rev. chron. order // if beforeid<0, same as db_get_messages struct db_message_list db_get_messages_before(i64 roomid,i64 count,i64 beforeid); +bool db_message_exists(i64 roomid,i64 msgid); void db_nullify_name_id(struct db_name_id ni); void db_nullify_room_list(struct db_room_list rl); diff --git a/event.c b/event.c index 40eaff8..e85eb45 100644 --- a/event.c +++ b/event.c @@ -67,13 +67,14 @@ void event_emit(const struct event_item *event){ } } -void event_emit_message(i64 timestamp,const char *message,const char *user,const char *room){ +void event_emit_message(i64 timestamp,const char *message,const char *user,const char *room,i64 replyid){ struct event_item *event=event_item_alloc(); event->type=EVENT_MESSAGE; event->timestamp=timestamp; event->message=strdup(message); event->user=strdup(user); event->room=strdup(room); + event->replyid=replyid; event_emit(event); event_item_free(event); } diff --git a/event.h b/event.h index 638cd20..0254702 100644 --- a/event.h +++ b/event.h @@ -13,6 +13,7 @@ struct event_item{ enum event_type type; i64 timestamp; // always set char *message,*user,*room; + i64 replyid; i64 num; }; @@ -26,6 +27,6 @@ struct event_item* event_item_alloc(void); void event_item_free(struct event_item *event); void event_emit(const struct event_item *event); -void event_emit_message(i64 timestamp,const char *message,const char *user,const char *room); +void event_emit_message(i64 timestamp,const char *message,const char *user,const char *room,i64 replyid); void event_emit_online(i64 timestamp,const char *user,i64 numonline); void event_emit_join(i64 timestamp,const char *user,const char *room); -- cgit v1.2.3-70-g09d2 From a9413077b5839b98fc791fcadbae4f8fa1c3743f Mon Sep 17 00:00:00 2001 From: Tom Smeding Date: Tue, 28 Jul 2020 16:05:57 +0200 Subject: protocol: Add get_message command for use with replies --- protocol.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/protocol.md b/protocol.md index 8c02017..8bbba49 100644 --- a/protocol.md +++ b/protocol.md @@ -63,6 +63,10 @@ A _response_ from the server will be in one of the following forms: the UNIX epoch. The reply message id is -1 if the message is a normal message, or nonnegative if it is a reply to the referenced message in the same room. +- ` message ` + - Response to the client's `get_message` command. This contains the + information for a single message, as in `_push message` and the + `history_message` response. A command is identified by its name, which can be found in the list below. Its arguments are zero or more words, except if for the command in question the last @@ -152,6 +156,9 @@ tomsg server. - Same as `history`, except the returned messages are the last `` strictly before the timestamp of the message with id ``. - Returns `history`, followed by zero or more `history_message`. +- ` get_message ` + - Retrieves the message with the given id. + - Returns `message` or `error`. - ` ping` - Asks for a `pong` response. - Returns `pong`. -- cgit v1.2.3-70-g09d2 From 1d28744e532ac73ddff0c6fa829949497b25d4b6 Mon Sep 17 00:00:00 2001 From: Tom Smeding Date: Tue, 28 Jul 2020 16:05:47 +0200 Subject: server: Implement get_message --- command.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- db.c | 37 +++++++++++++++++++++++++++++++------ db.h | 3 ++- 3 files changed, 85 insertions(+), 12 deletions(-) diff --git a/command.c b/command.c index ba8727e..ca70d76 100644 --- a/command.c +++ b/command.c @@ -257,9 +257,13 @@ static struct cmd_retval cmd_send(struct conn_data *data,const char *tag,const c net_send_error(data->fd,tag,"Not in that room"); return RET_OK; } - if(replyid>=0&&!db_message_exists(roomid,replyid)){ - net_send_error(data->fd,tag,"Replied-to message not found"); - return RET_OK; + if(replyid>=0){ + struct db_message msg=db_get_message(replyid); + if(msg.msgid==-1){ + net_send_error(data->fd,tag,"Replied-to message not found"); + return RET_OK; + } + db_nullify_message(msg); } i64 timestamp=make_timestamp(); @@ -361,6 +365,48 @@ static struct cmd_retval cmd_history_before(struct conn_data *data,const char *t return history_cmd_helper(data,tag,args,"history_before",beforeid); } +static struct cmd_retval cmd_get_message(struct conn_data *data, const char *tag, const char **args) { + i64 msgid; + if (!parse_i64(args[0], &msgid)) { + debug("Connection fd=%d sent an invalid number for 'get_message': '%s'", + data->fd, args[0]); + return RET_CLOSE(true); + } + + if (data->userid == -1) { + net_send_error(data->fd, tag, "Not logged in"); + return RET_OK; + } + + struct db_message msg = db_get_message(msgid); + if (msg.msgid == -1) { + net_send_error(data->fd, tag, "Message not found"); + return RET_OK; + } + + if (!db_is_member(msg.roomid, data->userid)) { + db_nullify_message(msg); + // Don't disclose that the message actually exists elsewhere + net_send_error(data->fd, tag, "Message not found"); + return RET_OK; + } + + char *roomname = db_get_roomname(msg.roomid); + char *username = db_get_username(msg.userid); + + char *buf = NULL; + i64 len = asprintf(&buf, "%s message %s %s %" PRIi64 " %" PRIi64 " %" PRIi64 " %s\n", + tag, roomname, username, msg.timestamp, msg.msgid, msg.replyid, msg.message); + free(roomname); + free(username); + + bool closed = net_send_raw_text(data->fd, buf, len); + free(buf); + + db_nullify_message(msg); + return RET_CLOSE(closed); +} + static struct cmd_retval cmd_ping(struct conn_data *data,const char *tag,const char **args){ (void)args; return RET_CLOSE(net_send_pong(data->fd,tag)); @@ -419,8 +465,8 @@ struct cmd_info{ // Use CommandHash.hs to re-generate this perfect hash function for a different // list of commands. -#define COMMAND_HASH_MODULUS 31 -#define COMMAND_HASH(cmd0, len) ((cmd0 + 6 * len) % COMMAND_HASH_MODULUS) +#define COMMAND_HASH_MODULUS 34 +#define COMMAND_HASH(cmd0, len) ((4 * cmd0 + 7 * len) % COMMAND_HASH_MODULUS) #define COMMAND_ENTRY(cmd0, cmd, nargs, longlast, handler) \ [COMMAND_HASH(cmd0, strlen(cmd))] = {cmd, nargs, longlast, handler} @@ -439,6 +485,7 @@ static const struct cmd_info commands[COMMAND_HASH_MODULUS] = { COMMAND_ENTRY('s', "send", 3, true, cmd_send), COMMAND_ENTRY('h', "history", 2, false, cmd_history), COMMAND_ENTRY('h', "history_before", 3, false, cmd_history_before), + COMMAND_ENTRY('g', "get_message", 1, false, cmd_get_message), COMMAND_ENTRY('p', "ping", 0, false, cmd_ping), COMMAND_ENTRY('i', "is_online", 1, false, cmd_is_online), COMMAND_ENTRY('f', "firebase_token", 1, false, cmd_firebase_token), diff --git a/db.c b/db.c index 416b9d3..d94b13c 100644 --- a/db.c +++ b/db.c @@ -460,16 +460,37 @@ struct db_message_list db_get_messages_before(i64 roomid,i64 count,i64 beforeid) return ml; } -bool db_message_exists(i64 roomid, i64 msgid) { +struct db_message db_get_message(i64 msgid) { sqlite3_stmt *stmt; SQLITE(prepare_v2, database, - "select id from Messages where id = ? and room = ?", + "select room, user, time, reply, message " + "from Messages where id = ?", -1, &stmt, NULL); SQLITE(bind_int64, stmt, 1, msgid); - SQLITE(bind_int64, stmt, 2, roomid); - bool success = sqlite3_step(stmt) == SQLITE_ROW; + + struct db_message msg; + if (sqlite3_step(stmt) == SQLITE_ROW) { + msg.msgid = msgid; + msg.roomid = sqlite3_column_int64(stmt,0); + msg.userid = sqlite3_column_int64(stmt,1); + msg.timestamp = sqlite3_column_int64(stmt,2); + if (sqlite3_column_type(stmt, 3) == SQLITE_INTEGER) { + msg.replyid = sqlite3_column_int64(stmt, 3); + } else { + msg.replyid = -1; // NULL, not a reply + } + msg.message = strdup((const char*)sqlite3_column_text(stmt, 4)); + } else { + msg.msgid = -1; + msg.roomid = -1; + msg.userid = -1; + msg.timestamp = -1; + msg.replyid = -1; + msg.message = NULL; + } + SQLITE(finalize, stmt); - return success; + return msg; } @@ -493,9 +514,13 @@ void db_nullify_user_list(struct db_user_list ul){ ul.list=NULL; } +void db_nullify_message(struct db_message msg){ + if(msg.message)free(msg.message); +} + void db_nullify_message_list(struct db_message_list ml){ for(i64 i=0;i Date: Tue, 28 Jul 2020 17:09:30 +0200 Subject: webclient: Show replies --- webclient/client.html | 83 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 15 deletions(-) diff --git a/webclient/client.html b/webclient/client.html index 28a91fd..512a2ed 100644 --- a/webclient/client.html +++ b/webclient/client.html @@ -4,12 +4,15 @@ tomsg webclient