Merge MAM improvements from #1724

I think this PR already solves and improves the MAM situation a lot.

What's @MarcoPolo-PasTonMolo still wanted to do in this branch is:
* MAM for mucs
* Check if url and quotes autocompletion works fine
* Check if the api still works fine
* Resolve conflicts

Conflicts are solved with this commit.

MAM for mucs can be another feature PR.

The rest we can check while being on master. And more people can help
testing.

src/database.c

@@ -46,6 +46,7 @@
 #include "log.h"
 #include "common.h"
 #include "config/files.h"
+#include "database.h"
 #include "xmpp/xmpp.h"
 #include "xmpp/message.h"
 
@@ -190,8 +191,9 @@ log_database_add_outgoing_muc_pm(const char* const id, const char* const barejid
     _log_database_add_outgoing("mucpm", id, barejid, message, replace_id, enc);
 }
 
-GSList*
+// Get info (timestamp and stanza_id) of the first or last message in db
-log_database_get_previous_chat(const gchar* const contact_barejid)
+ProfMessage*
+log_database_get_limits_info(const gchar* const contact_barejid, gboolean is_last)
 {
     sqlite3_stmt* stmt = NULL;
     gchar* query;
@@ -200,7 +202,63 @@ log_database_get_previous_chat(const gchar* const contact_barejid)
     if (!myjid)
         return NULL;
 
-    query = sqlite3_mprintf("SELECT * FROM (SELECT `message`, `timestamp`, `from_jid`, `type`, `encryption` from `ChatLogs` WHERE (`from_jid` = '%q' AND `to_jid` = '%q') OR (`from_jid` = '%q' AND `to_jid` = '%q') ORDER BY `timestamp` DESC LIMIT 10) ORDER BY `timestamp` ASC;", contact_barejid, myjid->barejid, myjid->barejid, contact_barejid);
+    if (is_last) {
+        query = sqlite3_mprintf("SELECT * FROM (SELECT `archive_id`, `timestamp` from `ChatLogs` WHERE (`from_jid` = '%q' AND `to_jid` = '%q') OR (`from_jid` = '%q' AND `to_jid` = '%q') ORDER BY `timestamp` DESC LIMIT 1) ORDER BY `timestamp` ASC;", contact_barejid, myjid->barejid, myjid->barejid, contact_barejid);
+    } else {
+        query = sqlite3_mprintf("SELECT * FROM (SELECT `archive_id`, `timestamp` from `ChatLogs` WHERE (`from_jid` = '%q' AND `to_jid` = '%q') OR (`from_jid` = '%q' AND `to_jid` = '%q') ORDER BY `timestamp` ASC LIMIT 1) ORDER BY `timestamp` ASC;", contact_barejid, myjid->barejid, myjid->barejid, contact_barejid);
+    }
+
+    if (!query) {
+        log_error("log_database_get_last_info(): SQL query. could not allocate memory");
+        return NULL;
+    }
+
+    jid_destroy(myjid);
+
+    int rc = sqlite3_prepare_v2(g_chatlog_database, query, -1, &stmt, NULL);
+    if (rc != SQLITE_OK) {
+        log_error("log_database_get_last_info(): unknown SQLite error");
+        return NULL;
+    }
+
+    ProfMessage* msg = message_init();
+
+    if (sqlite3_step(stmt) == SQLITE_ROW) {
+        char* archive_id = (char*)sqlite3_column_text(stmt, 0);
+        char* date = (char*)sqlite3_column_text(stmt, 1);
+
+        msg->stanzaid = strdup(archive_id);
+        msg->timestamp = g_date_time_new_from_iso8601(date, NULL);
+    }
+    sqlite3_finalize(stmt);
+    sqlite3_free(query);
+
+    return msg;
+}
+
+// Query previous chats, constraints start_time and end_time. If end_time is
+// null the current time is used. from_start gets first few messages if true
+// otherwise the last ones. Flip flips the order of the results
+GSList*
+log_database_get_previous_chat(const gchar* const contact_barejid, char* start_time, char* end_time, gboolean from_start, gboolean flip)
+{
+    sqlite3_stmt* stmt = NULL;
+    gchar* query;
+    const char* jid = connection_get_fulljid();
+    Jid* myjid = jid_create(jid);
+    if (!myjid)
+        return NULL;
+
+    // Flip order when querying older pages
+    gchar* sort1 = from_start ? "ASC" : "DESC";
+    gchar* sort2 = !flip ? "ASC" : "DESC";
+    GDateTime* now = g_date_time_new_now_local();
+    gchar* end_date_fmt = end_time ? end_time : g_date_time_format_iso8601(now);
+    query = sqlite3_mprintf("SELECT * FROM (SELECT COALESCE(B.`message`, A.`message`) AS message, A.`timestamp`, A.`from_jid`, A.`type` A.`encryption` from `ChatLogs` AS A LEFT JOIN `ChatLogs` AS B ON A.`stanza_id` = B.`replace_id` WHERE A.`replace_id` = '' AND ((A.`from_jid` = '%q' AND A.`to_jid` = '%q') OR (A.`from_jid` = '%q' AND A.`to_jid` = '%q')) AND A.`timestamp` < '%q' AND (%Q IS NULL OR A.`timestamp` > %Q) ORDER BY A.`timestamp` %s LIMIT %d) ORDER BY `timestamp` %s;", contact_barejid, myjid->barejid, myjid->barejid, contact_barejid, end_date_fmt, start_time, start_time, sort1, MESSAGES_TO_RETRIEVE, sort2);
+
+    g_date_time_unref(now);
+    g_free(end_date_fmt);
+
     if (!query) {
         log_error("log_database_get_previous_chat(): SQL query. could not allocate memory");
         return NULL;
@@ -330,7 +388,7 @@ _add_to_db(ProfMessage* message, char* type, const Jid* const from_jid, const Ji
         type = (char*)_get_message_type_str(message->type);
     }
 
-    query = sqlite3_mprintf("INSERT INTO `ChatLogs` (`from_jid`, `from_resource`, `to_jid`, `to_resource`, `message`, `timestamp`, `stanza_id`, `archive_id`, `replace_id`, `type`, `encryption`) SELECT '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q' WHERE NOT EXISTS (SELECT 1 FROM `ChatLogs` WHERE `archive_id` = '%q' AND `archive_id` != '')",
+    query = sqlite3_mprintf("INSERT INTO `ChatLogs` (`from_jid`, `from_resource`, `to_jid`, `to_resource`, `message`, `timestamp`, `stanza_id`, `archive_id`, `replace_id`, `type`, `encryption`) SELECT '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q' WHERE NOT EXISTS (SELECT 1 FROM `ChatLogs` WHERE (`archive_id` = '%q' AND `archive_id` != '') OR (`stanza_id` = '%q' AND `stanza_id` != ''))",
                             from_jid->barejid,
                             from_jid->resourcepart ? from_jid->resourcepart : "",
                             to_jid->barejid,
@@ -342,7 +400,8 @@ _add_to_db(ProfMessage* message, char* type, const Jid* const from_jid, const Ji
                             message->replace_id ? message->replace_id : "",
                             type ? type : "",
                             enc ? enc : "",
-                            message->stanzaid ? message->stanzaid : "");
+                            message->stanzaid ? message->stanzaid : "",
+                            message->id ? message->id : "");
     if (!query) {
         log_error("log_database_add(): SQL query. could not allocate memory");
         return;

src/database.h

@@ -40,12 +40,15 @@
 #include "config/account.h"
 #include "xmpp/xmpp.h"
 
+#define MESSAGES_TO_RETRIEVE 10
+
 gboolean log_database_init(ProfAccount* account);
 void log_database_add_incoming(ProfMessage* message);
 void log_database_add_outgoing_chat(const char* const id, const char* const barejid, const char* const message, const char* const replace_id, prof_enc_t enc);
 void log_database_add_outgoing_muc(const char* const id, const char* const barejid, const char* const message, const char* const replace_id, prof_enc_t enc);
 void log_database_add_outgoing_muc_pm(const char* const id, const char* const barejid, const char* const message, const char* const replace_id, prof_enc_t enc);
-GSList* log_database_get_previous_chat(const gchar* const contact_barejid);
+GSList* log_database_get_previous_chat(const gchar* const contact_barejid, char* start_time, char* end_time, gboolean from_start, gboolean flip);
+ProfMessage* log_database_get_limits_info(const gchar* const contact_barejid, gboolean is_last);
 void log_database_close(void);
 
 #endif // DATABASE_H

src/event/server_events.c

@@ -55,6 +55,7 @@
 #include "event/common.h"
 #include "plugins/plugins.h"
 #include "ui/window_list.h"
+#include "ui/window.h"
 #include "tools/bookmark_ignore.h"
 #include "xmpp/xmpp.h"
 #include "xmpp/muc.h"
@@ -648,6 +649,11 @@ sv_ev_incoming_message(ProfMessage* message)
         chatwin = (ProfChatWin*)window;
         new_win = TRUE;
 
+        if (prefs_get_boolean(PREF_MAM)) {
+            win_print_loading_history(window);
+            iq_mam_request(chatwin, g_date_time_add_seconds(message->timestamp, 0)); // copy timestamp
+        }
+
 #ifdef HAVE_OMEMO
         if (!message->is_mam) {
             if (omemo_automatic_start(message->from_jid->barejid)) {

src/ui/buffer.c

@@ -110,6 +110,34 @@ buffer_append(ProfBuff buffer, const char* show_char, int pad_indent, GDateTime*
     buffer->entries = g_slist_append(buffer->entries, e);
 }
 
+void
+buffer_prepend(ProfBuff buffer, const char* show_char, int pad_indent, GDateTime* time, int flags, theme_item_t theme_item, const char* const display_from, const char* const from_jid, const char* const message, DeliveryReceipt* receipt, const char* const id)
+{
+    ProfBuffEntry* e = malloc(sizeof(struct prof_buff_entry_t));
+    e->show_char = strdup(show_char);
+    e->pad_indent = pad_indent;
+    e->flags = flags;
+    e->theme_item = theme_item;
+    e->time = g_date_time_ref(time);
+    e->display_from = display_from ? strdup(display_from) : NULL;
+    e->from_jid = from_jid ? strdup(from_jid) : NULL;
+    e->message = strdup(message);
+    e->receipt = receipt;
+    if (id) {
+        e->id = strdup(id);
+    } else {
+        e->id = NULL;
+    }
+
+    if (g_slist_length(buffer->entries) == BUFF_SIZE) {
+        GSList* last = g_slist_last(buffer->entries);
+        _free_entry(last->data);
+        buffer->entries = g_slist_delete_link(buffer->entries, last);
+    }
+
+    buffer->entries = g_slist_prepend(buffer->entries, e);
+}
+
 void
 buffer_remove_entry_by_id(ProfBuff buffer, const char* const id)
 {
@@ -125,6 +153,14 @@ buffer_remove_entry_by_id(ProfBuff buffer, const char* const id)
     }
 }
 
+void
+buffer_remove_entry(ProfBuff buffer, int entry)
+{
+    GSList* node = g_slist_nth(buffer->entries, entry);
+    _free_entry(node->data);
+    buffer->entries = g_slist_delete_link(buffer->entries, node);
+}
+
 gboolean
 buffer_mark_received(ProfBuff buffer, const char* const id)
 {

src/ui/buffer.h

@@ -70,7 +70,9 @@ typedef struct prof_buff_t* ProfBuff;
 ProfBuff buffer_create();
 void buffer_free(ProfBuff buffer);
 void buffer_append(ProfBuff buffer, const char* show_char, int pad_indent, GDateTime* time, int flags, theme_item_t theme_item, const char* const display_from, const char* const barejid, const char* const message, DeliveryReceipt* receipt, const char* const id);
+void buffer_prepend(ProfBuff buffer, const char* show_char, int pad_indent, GDateTime* time, int flags, theme_item_t theme_item, const char* const display_from, const char* const barejid, const char* const message, DeliveryReceipt* receipt, const char* const id);
 void buffer_remove_entry_by_id(ProfBuff buffer, const char* const id);
+void buffer_remove_entry(ProfBuff buffer, int entry);
 int buffer_size(ProfBuff buffer);
 ProfBuffEntry* buffer_get_entry(ProfBuff buffer, int entry);
 ProfBuffEntry* buffer_get_entry_by_id(ProfBuff buffer, const char* const id);

src/ui/chatwin.c

@@ -96,7 +96,7 @@ chatwin_new(const char* const barejid)
     ProfWin* window = wins_new_chat(barejid);
     ProfChatWin* chatwin = (ProfChatWin*)window;
 
-    if (!prefs_get_boolean(PREF_MAM) && prefs_get_boolean(PREF_CHLOG) && prefs_get_boolean(PREF_HISTORY)) {
+    if (!prefs_get_boolean(PREF_MAM) && (prefs_get_boolean(PREF_CHLOG) && prefs_get_boolean(PREF_HISTORY))) {
         _chatwin_history(chatwin, barejid);
     }
 
@@ -146,7 +146,8 @@ chatwin_new(const char* const barejid)
     }
 
     if (prefs_get_boolean(PREF_MAM)) {
-        iq_mam_request(chatwin);
+        iq_mam_request(chatwin, NULL);
+        win_print_loading_history(window);
     }
 
     return chatwin;
@@ -333,7 +334,7 @@ chatwin_incoming_msg(ProfChatWin* chatwin, ProfMessage* message, gboolean win_cr
     free(mybarejid);
 
     gboolean is_current = wins_is_current(window);
-    gboolean notify = prefs_do_chat_notify(is_current);
+    gboolean notify = prefs_do_chat_notify(is_current) && !message->is_mam;
 
     // currently viewing chat window with sender
     if (wins_is_current(window)) {
@@ -344,14 +345,17 @@ chatwin_incoming_msg(ProfChatWin* chatwin, ProfMessage* message, gboolean win_cr
         // not currently viewing chat window with sender
     } else {
         status_bar_new(num, WIN_CHAT, chatwin->barejid);
-        cons_show_incoming_message(display_name, num, chatwin->unread, window);
 
-        if (prefs_get_boolean(PREF_FLASH)) {
+        if (!message->is_mam) {
-            flash();
+            cons_show_incoming_message(display_name, num, chatwin->unread, window);
+
+            if (prefs_get_boolean(PREF_FLASH)) {
+                flash();
+            }
+
+            chatwin->unread++;
         }
 
-        chatwin->unread++;
-
         // TODO: so far we don't ask for MAM when incoming message occurs.
         // Need to figure out:
         // 1) only send IQ once
@@ -379,7 +383,7 @@ chatwin_incoming_msg(ProfChatWin* chatwin, ProfMessage* message, gboolean win_cr
     wins_add_urls_ac(window, message);
     wins_add_quotes_ac(window, message->plain);
 
-    if (prefs_get_boolean(PREF_BEEP)) {
+    if (prefs_get_boolean(PREF_BEEP) && !message->is_mam) {
         beep();
     }
 
@@ -572,7 +576,7 @@ static void
 _chatwin_history(ProfChatWin* chatwin, const char* const contact_barejid)
 {
     if (!chatwin->history_shown) {
-        GSList* history = log_database_get_previous_chat(contact_barejid);
+        GSList* history = log_database_get_previous_chat(contact_barejid, NULL, NULL, FALSE, FALSE);
         GSList* curr = history;
 
         while (curr) {
@@ -591,6 +595,41 @@ _chatwin_history(ProfChatWin* chatwin, const char* const contact_barejid)
     }
 }
 
+// Print history starting from start_time to end_time if end_time is null the
+// first entry's timestamp in the buffer is used. Flip true to prepend to buffer.
+// Timestamps should be in iso8601
+gboolean
+chatwin_db_history(ProfChatWin* chatwin, char* start_time, char* end_time, gboolean flip)
+{
+    if (!end_time) {
+        end_time = buffer_size(((ProfWin*)chatwin)->layout->buffer) == 0 ? NULL : g_date_time_format_iso8601(buffer_get_entry(((ProfWin*)chatwin)->layout->buffer, 0)->time);
+    }
+
+    GSList* history = log_database_get_previous_chat(chatwin->barejid, start_time, end_time, !flip, flip);
+    gboolean has_items = g_slist_length(history) != 0;
+    GSList* curr = history;
+
+    while (curr) {
+        ProfMessage* msg = curr->data;
+        char* msg_plain = msg->plain;
+        msg->plain = plugins_pre_chat_message_display(msg->from_jid->barejid, msg->from_jid->resourcepart, msg->plain);
+        // This is dirty workaround for memory leak. We reassign msg->plain above so have to free previous object
+        // TODO: Make a better solution, for example, pass msg object to the function and it will replace msg->plain properly if needed.
+        free(msg_plain);
+        if (flip) {
+            win_print_old_history((ProfWin*)chatwin, msg);
+        } else {
+            win_print_history((ProfWin*)chatwin, msg);
+        }
+        curr = g_slist_next(curr);
+    }
+
+    g_slist_free_full(history, (GDestroyNotify)message_free);
+    win_redraw((ProfWin*)chatwin);
+
+    return has_items;
+}
+
 static void
 _chatwin_set_last_message(ProfChatWin* chatwin, const char* const id, const char* const message)
 {

src/ui/ui.h

@@ -145,6 +145,7 @@ void chatwin_set_incoming_char(ProfChatWin* chatwin, const char* const ch);
 void chatwin_unset_incoming_char(ProfChatWin* chatwin);
 void chatwin_set_outgoing_char(ProfChatWin* chatwin, const char* const ch);
 void chatwin_unset_outgoing_char(ProfChatWin* chatwin);
+gboolean chatwin_db_history(ProfChatWin* chatwin, char* start_time, char* end_time, gboolean flip);
 
 // MUC window
 ProfMucWin* mucwin_new(const char* const barejid);

src/ui/window.c

@@ -61,6 +61,7 @@
 #include "xmpp/xmpp.h"
 #include "xmpp/roster_list.h"
 #include "xmpp/connection.h"
+#include "database.h"
 
 #define CONS_WIN_TITLE "Profanity. Type /help for help information."
 #define XML_WIN_TITLE  "XML Console"
@@ -637,6 +638,19 @@ win_page_up(ProfWin* window)
 
     *page_start -= page_space;
 
+    if (*page_start == -page_space && window->type == WIN_CHAT) {
+        ProfChatWin* chatwin = (ProfChatWin*) window;
+        ProfBuffEntry* first_entry = buffer_size(window->layout->buffer) != 0 ? buffer_get_entry(window->layout->buffer, 0) : NULL;
+
+        // Don't do anything if still fetching mam messages
+        if (first_entry && !(first_entry->theme_item == THEME_ROOMINFO && g_strcmp0(first_entry->message, LOADING_MESSAGE) == 0)) {
+            if (!chatwin_db_history(chatwin, NULL, NULL, TRUE) && prefs_get_boolean(PREF_MAM)) {
+                win_print_loading_history(window);
+                iq_mam_request_older(chatwin);
+            }
+        }
+    }
+
     // went past beginning, show first page
     if (*page_start < 0)
         *page_start = 0;
@@ -660,6 +674,20 @@ win_page_down(ProfWin* window)
 
     *page_start += page_space;
 
+    // Scrolled down after reaching the bottom of the page
+    if ((*page_start == y || (*page_start == page_space && *page_start >= y)) && window->type == WIN_CHAT) {
+        int bf_size = buffer_size(window->layout->buffer);
+        if (bf_size > 0) {
+            char* start = g_date_time_format_iso8601(buffer_get_entry(window->layout->buffer, bf_size - 1)->time);
+            GDateTime* now = g_date_time_new_now_local();
+            char* end = g_date_time_format_iso8601(now);
+            chatwin_db_history((ProfChatWin*)window, start, end, FALSE);
+
+            g_free(start);
+            g_date_time_unref(now);
+        }
+    }
+
     // only got half a screen, show full screen
     if ((y - (*page_start)) < page_space)
         *page_start = y - page_space;
@@ -1386,8 +1414,8 @@ win_print_incoming(ProfWin* window, const char* const display_name_from, ProfMes
         if (prefs_get_boolean(PREF_CORRECTION_ALLOW) && message->replace_id) {
             _win_correct(window, message->plain, message->id, message->replace_id, message->from_jid->barejid);
         } else {
-            // Prevent duplicate messages when current client is sending a message
+            // Prevent duplicate messages when current client is sending a message or if it's mam
-            if (g_strcmp0(message->from_jid->fulljid, connection_get_fulljid()) != 0) {
+            if (g_strcmp0(message->from_jid->fulljid, connection_get_fulljid()) != 0 && !message->is_mam) {
                 _win_printf(window, enc_char, 0, message->timestamp, flags, THEME_TEXT_THEM, display_name_from, message->from_jid->barejid, message->id, "%s", message->plain);
             }
         }
@@ -1482,6 +1510,34 @@ win_print_history(ProfWin* window, const ProfMessage* const message)
     g_date_time_unref(message->timestamp);
 }
 
+void
+win_print_old_history(ProfWin* window, const ProfMessage* const message)
+{
+    g_date_time_ref(message->timestamp);
+
+    char* display_name;
+    int flags = 0;
+    const char* jid = connection_get_fulljid();
+    Jid* jidp = jid_create(jid);
+
+    if (g_strcmp0(jidp->barejid, message->from_jid->barejid) == 0) {
+        display_name = strdup("me");
+    } else {
+        display_name = roster_get_msg_display_name(message->from_jid->barejid, message->from_jid->resourcepart);
+        flags = NO_ME;
+    }
+
+    jid_destroy(jidp);
+
+    buffer_prepend(window->layout->buffer, "-", 0, message->timestamp, flags, THEME_TEXT_HISTORY, display_name, NULL, message->plain, NULL, NULL);
+    _win_print_internal(window, "-", 0, message->timestamp, flags, THEME_TEXT_HISTORY, display_name, message->plain, NULL);
+
+    free(display_name);
+
+    inp_nonblocking(TRUE);
+    g_date_time_unref(message->timestamp);
+}
+
 void
 win_print(ProfWin* window, theme_item_t theme_item, const char* show_char, const char* const message, ...)
 {
@@ -1698,7 +1754,6 @@ _win_printf(ProfWin* window, const char* show_char, int pad_indent, GDateTime* t
     g_string_vprintf(fmt_msg, message, arg);
 
     buffer_append(window->layout->buffer, show_char, pad_indent, timestamp, flags, theme_item, display_from, from_jid, fmt_msg->str, NULL, message_id);
-
     _win_print_internal(window, show_char, pad_indent, timestamp, flags, theme_item, display_from, fmt_msg->str, NULL);
 
     inp_nonblocking(TRUE);
@@ -2002,6 +2057,15 @@ win_redraw(ProfWin* window)
     }
 }
 
+void
+win_print_loading_history(ProfWin* window)
+{
+    GDateTime* timestamp = buffer_size(window->layout->buffer) != 0 ? buffer_get_entry(window->layout->buffer, 0)->time : g_date_time_new_now_local();
+    buffer_prepend(window->layout->buffer, "-", 0, timestamp, NO_DATE, THEME_ROOMINFO, NULL, NULL, LOADING_MESSAGE, NULL, NULL);
+    g_date_time_unref(timestamp);
+    win_redraw(window);
+}
+
 gboolean
 win_has_active_subwin(ProfWin* window)
 {

src/ui/window.h

@@ -57,6 +57,7 @@
 #include "xmpp/muc.h"
 
 #define PAD_SIZE 1000
+#define LOADING_MESSAGE "Loading older messages ..."
 
 void win_move_to_end(ProfWin* window);
 void win_show_status_string(ProfWin* window, const char* const from,
@@ -71,11 +72,13 @@ void win_print_outgoing_with_receipt(ProfWin* window, const char* show_char, con
 void win_println_incoming_muc_msg(ProfWin* window, char* show_char, int flags, const ProfMessage* const message);
 void win_print_outgoing_muc_msg(ProfWin* window, char* show_char, const char* const me, const char* const id, const char* const replace_id, const char* const message);
 void win_print_history(ProfWin* window, const ProfMessage* const message);
+void win_print_old_history(ProfWin* window, const ProfMessage* const message);
 
 void win_print_http_transfer(ProfWin* window, const char* const message, char* url);
 
 void win_newline(ProfWin* window);
 void win_redraw(ProfWin* window);
+void win_print_loading_history(ProfWin* window);
 int win_roster_cols(void);
 int win_occpuants_cols(void);
 void win_sub_print(WINDOW* win, char* msg, gboolean newline, gboolean wrap, int indent);

src/xmpp/iq.c

@@ -66,6 +66,8 @@
 #include "xmpp/roster_list.h"
 #include "xmpp/roster.h"
 #include "xmpp/muc.h"
+#include "src/database.h"
+#include "ui/window.h"
 
 #ifdef HAVE_OMEMO
 #include "omemo/omemo.h"
@@ -105,9 +107,19 @@ typedef struct command_config_data_t
 typedef struct mam_rsm_userdata
 {
     char* barejid;
-    char* datestr;
+    char* start_datestr;
+    char* end_datestr;
+    gboolean fetch_next;
+    ProfChatWin* win;
 } MamRsmUserdata;
 
+typedef struct late_delivery_userdata
+{
+    ProfChatWin* win;
+    GDateTime* enddate;
+    GDateTime* startdate;
+} LateDeliveryUserdata;
+
 static int _iq_handler(xmpp_conn_t* const conn, xmpp_stanza_t* const stanza, void* const userdata);
 
 static void _error_handler(xmpp_stanza_t* const stanza);
@@ -145,6 +157,7 @@ static int _command_exec_response_handler(xmpp_stanza_t* const stanza, void* con
 static int _mam_rsm_id_handler(xmpp_stanza_t* const stanza, void* const userdata);
 static int _register_change_password_result_id_handler(xmpp_stanza_t* const stanza, void* const userdata);
 
+static void _iq_mam_request(ProfChatWin* win, GDateTime* startdate, GDateTime* enddate);
 static void _iq_free_room_data(ProfRoomInfoData* roominfo);
 static void _iq_free_affiliation_set(ProfPrivilegeSet* affiliation_set);
 static void _iq_free_affiliation_list(ProfAffiliationList* affiliation_list);
@@ -160,6 +173,8 @@ static gboolean autoping_wait = FALSE;
 static GTimer* autoping_time = NULL;
 static GHashTable* id_handlers;
 static GHashTable* rooms_cache = NULL;
+static GSList* late_delivery_windows = NULL;
+static gboolean received_disco_items = FALSE;
 
 static int
 _iq_handler(xmpp_conn_t* const conn, xmpp_stanza_t* const stanza, void* const userdata)
@@ -254,6 +269,8 @@ iq_handlers_init(void)
 
     id_handlers = g_hash_table_new_full(g_str_hash, g_str_equal, free, (GDestroyNotify)_iq_id_handler_free);
     rooms_cache = g_hash_table_new_full(g_str_hash, g_str_equal, free, (GDestroyNotify)xmpp_stanza_release);
+    late_delivery_windows = malloc(sizeof(GSList *));
+    late_delivery_windows->data = NULL;
 }
 
 void
@@ -2521,7 +2538,16 @@ _disco_items_result_handler(xmpp_stanza_t* const stanza)
     if (g_strcmp0(id, "discoitemsreq") == 0) {
         cons_show_disco_items(items, from);
     } else if (g_strcmp0(id, "discoitemsreq_onconnect") == 0) {
+        received_disco_items = TRUE;
         connection_set_disco_items(items);
+
+        while (late_delivery_windows->data) {
+            LateDeliveryUserdata* del_data = late_delivery_windows->data;
+            _iq_mam_request(del_data->win, del_data->startdate, del_data->enddate);
+
+            late_delivery_windows = g_slist_next(late_delivery_windows);
+            free(del_data);
+        }
     }
 
     g_slist_free_full(items, (GDestroyNotify)_item_destroy);
@@ -2573,8 +2599,18 @@ _iq_free_affiliation_list(ProfAffiliationList* affiliation_list)
     }
 }
 
+static int
+_mam_buffer_commit_handler(xmpp_stanza_t* const stanza, void* const userdata)
+{
+    ProfChatWin* chatwin = (ProfChatWin*)userdata;
+    // Remove the "Loading messages ..." message
+    buffer_remove_entry(((ProfWin*)chatwin)->layout->buffer, 0);
+    chatwin_db_history(chatwin, NULL, NULL, TRUE);
+    return 0;
+}
+
 void
-iq_mam_request(ProfChatWin* win)
+iq_mam_request_older(ProfChatWin* win)
 {
     if (connection_supports(XMPP_FEATURE_MAM2) == FALSE) {
         log_warning("Server doesn't advertise %s feature.", XMPP_FEATURE_MAM2);
@@ -2582,31 +2618,106 @@ iq_mam_request(ProfChatWin* win)
         return;
     }
 
+    ProfMessage* first_msg = log_database_get_limits_info(win->barejid, FALSE);
+    char* firstid = NULL;
+    char* enddate = NULL;
+
+    // If first message found
+    if (first_msg->timestamp) {
+        firstid = first_msg->stanzaid;
+        enddate = g_date_time_format(first_msg->timestamp, "%FT%T.%f%:z");
+    } else {
+        return;
+    }
+
+    xmpp_ctx_t* const ctx = connection_get_ctx();
+    xmpp_stanza_t* iq = stanza_create_mam_iq(ctx, win->barejid, NULL, enddate, firstid, NULL);
+    iq_id_handler_add(xmpp_stanza_get_id(iq), _mam_buffer_commit_handler, NULL, win);
+
+    g_free(enddate);
+    message_free(first_msg);
+
+    iq_send_stanza(iq);
+    xmpp_stanza_release(iq);
+
+    return;
+}
+
+void
+_iq_mam_request(ProfChatWin* win, GDateTime* startdate, GDateTime* enddate)
+{
+    if (connection_supports(XMPP_FEATURE_MAM2) == FALSE) {
+        log_warning("Server doesn't advertise %s feature.", XMPP_FEATURE_MAM2);
+        cons_show_error("Server doesn't support MAM (%s).", XMPP_FEATURE_MAM2);
+        return;
+    }
+
+    char* firstid = "";
+    char* startdate_str = NULL;
+    char* enddate_str = NULL;
+    gboolean fetch_next = FALSE;
+
+    if (startdate) {
+        startdate_str = g_date_time_format(startdate, "%FT%T.%f%:z");
+        fetch_next = TRUE;
+        g_date_time_unref(startdate);
+    } else if (!enddate) {
+        GDateTime* now = g_date_time_new_now_utc();
+        enddate_str = g_date_time_format(now, "%FT%T.%f%:z");
+        g_date_time_unref(now);
+    }
+
+    if (enddate) {
+        enddate_str = g_date_time_format(enddate, "%FT%T.%f%:z");
+        g_date_time_unref(enddate);
+    }
+
     xmpp_ctx_t* const ctx = connection_get_ctx();
 
-    GDateTime* now = g_date_time_new_now_utc();
+    xmpp_stanza_t* iq = stanza_create_mam_iq(ctx, win->barejid, startdate_str, enddate_str, firstid, NULL);
-    GDateTime* timestamp = g_date_time_add_days(now, -7);
-    g_date_time_unref(now);
-    gchar* datestr = g_date_time_format(timestamp, "%FT%TZ");
-    xmpp_stanza_t* iq = stanza_create_mam_iq(ctx, win->barejid, datestr, NULL);
 
     MamRsmUserdata* data = malloc(sizeof(MamRsmUserdata));
     if (data) {
-        data->datestr = strdup(datestr);
+        data->start_datestr = startdate_str;
+        data->end_datestr = enddate_str;
         data->barejid = strdup(win->barejid);
+        data->fetch_next = fetch_next;
+        data->win = win;
 
         iq_id_handler_add(xmpp_stanza_get_id(iq), _mam_rsm_id_handler, NULL, data);
     }
 
-    g_free(datestr);
-    g_date_time_unref(timestamp);
-
     iq_send_stanza(iq);
     xmpp_stanza_release(iq);
 
     return;
 }
 
+void
+iq_mam_request(ProfChatWin* win, GDateTime* enddate)
+{
+    ProfMessage* last_msg = log_database_get_limits_info(win->barejid, TRUE);
+    GDateTime* startdate = last_msg->timestamp ? g_date_time_add_seconds(last_msg->timestamp, 0) : NULL; // copy timestamp
+    message_free(last_msg);
+
+    // Save request for later if disco items haven't been received yet
+    if (!received_disco_items) {
+        if (late_delivery_windows->data == NULL) {
+            LateDeliveryUserdata* cur_del_data = malloc(sizeof(LateDeliveryUserdata));
+            cur_del_data->win = win;
+            cur_del_data->enddate = enddate;
+            cur_del_data->startdate = startdate;
+            late_delivery_windows->data = cur_del_data;
+        }
+        late_delivery_windows = g_slist_append(late_delivery_windows, NULL);
+    }
+
+
+    _iq_mam_request(win, startdate, enddate);
+
+    return;
+}
+
 static int
 _mam_rsm_id_handler(xmpp_stanza_t* const stanza, void* const userdata)
 {
@@ -2619,23 +2730,64 @@ _mam_rsm_id_handler(xmpp_stanza_t* const stanza, void* const userdata)
     } else if (g_strcmp0(type, "result") == 0) {
         xmpp_stanza_t* fin = xmpp_stanza_get_child_by_name_and_ns(stanza, STANZA_NAME_FIN, STANZA_NS_MAM2);
         if (fin) {
+            gboolean is_complete = g_strcmp0(xmpp_stanza_get_attribute(fin, "complete"), "true") == 0;
+            MamRsmUserdata* data = (MamRsmUserdata*)userdata;
+            ProfWin* window = (ProfWin*)data->win;
+
+            buffer_remove_entry(window->layout->buffer, 0);
+
+            char *start_str = NULL;
+            if (data->start_datestr) {
+                start_str = strdup(data->start_datestr);
+                // Convert to iso8601
+                start_str[strlen(start_str) - 3] = '\0';
+            }
+            char *end_str = NULL;
+            if (data->end_datestr) {
+                end_str = strdup(data->end_datestr);
+                // Convert to iso8601
+                end_str[strlen(end_str) - 3] = '\0';
+            }
+
+            if (is_complete || !data->fetch_next) {
+                chatwin_db_history(data->win, is_complete ? NULL : start_str, end_str, TRUE);
+                // TODO free memory
+                if (start_str) {
+                    free(start_str);
+                    free(data->start_datestr);
+                }
+
+                if (end_str) {
+                    free(data->end_datestr);
+                }
+
+                free(data->barejid);
+                free(data);
+                return 0;
+            }
+
+            chatwin_db_history(data->win, start_str, end_str, TRUE);
+            if (start_str) free(start_str);
+
             xmpp_stanza_t* set = xmpp_stanza_get_child_by_name_and_ns(fin, STANZA_TYPE_SET, STANZA_NS_RSM);
             if (set) {
-                char* lastid = NULL;
+                win_print_loading_history(window);
-                xmpp_stanza_t* last = xmpp_stanza_get_child_by_name(set, STANZA_NAME_LAST);
+
-                if (last) {
+                char* firstid = NULL;
-                    lastid = xmpp_stanza_get_text(last);
+                xmpp_stanza_t* first = xmpp_stanza_get_child_by_name(set, STANZA_NAME_FIRST);
-                }
+                firstid = xmpp_stanza_get_text(first);
 
                 // 4.3.2. send same stanza with set,max stanza
                 xmpp_ctx_t* const ctx = connection_get_ctx();
 
-                MamRsmUserdata* data = (MamRsmUserdata*)userdata;
+                if (end_str) {
-                xmpp_stanza_t* iq = stanza_create_mam_iq(ctx, data->barejid, data->datestr, lastid);
+                    free(data->end_datestr);
-                free(data->barejid);
+                }
-                free(data->datestr);
+                data->end_datestr = NULL;
-                free(data);
+                xmpp_stanza_t* iq = stanza_create_mam_iq(ctx, data->barejid, data->start_datestr, data->end_datestr, firstid, NULL);
-                free(lastid);
+                free(firstid);
+
+                iq_id_handler_add(xmpp_stanza_get_id(iq), _mam_rsm_id_handler, NULL, data);
 
                 iq_send_stanza(iq);
                 xmpp_stanza_release(iq);

src/xmpp/stanza.c

@@ -58,6 +58,7 @@
 #include "xmpp/connection.h"
 #include "xmpp/form.h"
 #include "xmpp/muc.h"
+#include "database.h"
 
 static void _stanza_add_unique_id(xmpp_stanza_t* stanza);
 static char* _stanza_create_sha1_hash(char* str);
@@ -2720,7 +2721,7 @@ stanza_attach_correction(xmpp_ctx_t* ctx, xmpp_stanza_t* stanza, const char* con
 }
 
 xmpp_stanza_t*
-stanza_create_mam_iq(xmpp_ctx_t* ctx, const char* const jid, const char* const startdate, const char* const lastid)
+stanza_create_mam_iq(xmpp_ctx_t* ctx, const char* const jid, const char* const startdate, const char* const enddate, const char* const firstid, const char* const lastid)
 {
     char* id = connection_create_stanza_id();
     xmpp_stanza_t* iq = xmpp_iq_new(ctx, STANZA_TYPE_SET, id);
@@ -2766,26 +2767,57 @@ stanza_create_mam_iq(xmpp_ctx_t* ctx, const char* const jid, const char* const s
     xmpp_stanza_add_child(field_with, value_with);
 
     // field 'start'
-    xmpp_stanza_t* field_start = xmpp_stanza_new(ctx);
+    xmpp_stanza_t* field_start, *value_start, *start_date_text;
-    xmpp_stanza_set_name(field_start, STANZA_NAME_FIELD);
+    if (startdate) {
-    xmpp_stanza_set_attribute(field_start, STANZA_ATTR_VAR, "start");
+        field_start = xmpp_stanza_new(ctx);
+        xmpp_stanza_set_name(field_start, STANZA_NAME_FIELD);
+        xmpp_stanza_set_attribute(field_start, STANZA_ATTR_VAR, "start");
 
-    xmpp_stanza_t* value_start = xmpp_stanza_new(ctx);
+        value_start = xmpp_stanza_new(ctx);
-    xmpp_stanza_set_name(value_start, STANZA_NAME_VALUE);
+        xmpp_stanza_set_name(value_start, STANZA_NAME_VALUE);
 
-    xmpp_stanza_t* date_text = xmpp_stanza_new(ctx);
+        start_date_text = xmpp_stanza_new(ctx);
-    xmpp_stanza_set_text(date_text, startdate);
+        xmpp_stanza_set_text(start_date_text, startdate);
-    xmpp_stanza_add_child(value_start, date_text);
+        xmpp_stanza_add_child(value_start, start_date_text);
 
-    xmpp_stanza_add_child(field_start, value_start);
+        xmpp_stanza_add_child(field_start, value_start);
+    }
+
+    xmpp_stanza_t* field_end, *value_end, *end_date_text;
+    if (enddate) {
+        field_end = xmpp_stanza_new(ctx);
+        xmpp_stanza_set_name(field_end, STANZA_NAME_FIELD);
+        xmpp_stanza_set_attribute(field_end, STANZA_ATTR_VAR, "end");
+
+        value_end = xmpp_stanza_new(ctx);
+        xmpp_stanza_set_name(value_end, STANZA_NAME_VALUE);
+
+        end_date_text = xmpp_stanza_new(ctx);
+        xmpp_stanza_set_text(end_date_text, enddate);
+        xmpp_stanza_add_child(value_end, end_date_text);
+
+        xmpp_stanza_add_child(field_end, value_end);
+    }
 
     // 4.3.2 set/rsm
-    xmpp_stanza_t *after, *after_text, *set;
+    xmpp_stanza_t *set, *max, *max_text;
-    if (lastid) {
+    set = xmpp_stanza_new(ctx);
-        set = xmpp_stanza_new(ctx);
+    xmpp_stanza_set_name(set, STANZA_TYPE_SET);
-        xmpp_stanza_set_name(set, STANZA_TYPE_SET);
+    xmpp_stanza_set_ns(set, STANZA_NS_RSM);
-        xmpp_stanza_set_ns(set, STANZA_NS_RSM);
 
+    max = xmpp_stanza_new(ctx);
+    xmpp_stanza_set_name(max, STANZA_NAME_MAX);
+
+    max_text = xmpp_stanza_new(ctx);
+    char* txt = g_strdup_printf("%d", MESSAGES_TO_RETRIEVE);
+    xmpp_stanza_set_text(max_text, txt);
+    g_free(txt);
+
+    xmpp_stanza_add_child(max, max_text);
+    xmpp_stanza_add_child(set, max);
+
+    xmpp_stanza_t *after, *after_text;
+    if (lastid) {
         after = xmpp_stanza_new(ctx);
         xmpp_stanza_set_name(after, STANZA_NAME_AFTER);
 
@@ -2796,30 +2828,59 @@ stanza_create_mam_iq(xmpp_ctx_t* ctx, const char* const jid, const char* const s
         xmpp_stanza_add_child(set, after);
     }
 
+    xmpp_stanza_t *before, *before_text;
+    if (firstid) {
+        before = xmpp_stanza_new(ctx);
+        xmpp_stanza_set_name(before, STANZA_NAME_BEFORE);
+
+        before_text = xmpp_stanza_new(ctx);
+        xmpp_stanza_set_text(before_text, firstid);
+
+        xmpp_stanza_add_child(before, before_text);
+        xmpp_stanza_add_child(set, before);
+    }
+
     // add and release
     xmpp_stanza_add_child(iq, query);
     xmpp_stanza_add_child(query, x);
     xmpp_stanza_add_child(x, field_form_type);
     xmpp_stanza_add_child(x, field_with);
-    xmpp_stanza_add_child(x, field_start);
+
+    if (startdate) {
+        xmpp_stanza_add_child(x, field_start);
+        xmpp_stanza_release(field_start);
+        xmpp_stanza_release(value_start);
+        xmpp_stanza_release(start_date_text);
+    }
+
+    if (enddate) {
+        xmpp_stanza_add_child(x, field_end);
+        xmpp_stanza_release(field_end);
+        xmpp_stanza_release(value_end);
+        xmpp_stanza_release(end_date_text);
+    }
+
+    xmpp_stanza_add_child(query, set);
+    xmpp_stanza_release(set);
+    xmpp_stanza_release(max_text);
+    xmpp_stanza_release(max);
 
     if (lastid) {
-        xmpp_stanza_add_child(query, after);
-
         xmpp_stanza_release(after_text);
         xmpp_stanza_release(after);
-        xmpp_stanza_release(set);
+    }
+
+    if (firstid) {
+        xmpp_stanza_release(before_text);
+        xmpp_stanza_release(before);
     }
 
     xmpp_stanza_release(mam_text);
     xmpp_stanza_release(with_text);
-    xmpp_stanza_release(date_text);
     xmpp_stanza_release(value_mam);
     xmpp_stanza_release(value_with);
-    xmpp_stanza_release(value_start);
     xmpp_stanza_release(field_form_type);
     xmpp_stanza_release(field_with);
-    xmpp_stanza_release(field_start);
     xmpp_stanza_release(x);
     xmpp_stanza_release(query);
 

src/xmpp/stanza.h

@@ -115,7 +115,10 @@
 #define STANZA_NAME_MINIMIZE         "minimize"
 #define STANZA_NAME_FIN              "fin"
 #define STANZA_NAME_LAST             "last"
+#define STANZA_NAME_FIRST            "first"
 #define STANZA_NAME_AFTER            "after"
+#define STANZA_NAME_BEFORE           "before"
+#define STANZA_NAME_MAX              "max"
 #define STANZA_NAME_USERNAME         "username"
 #define STANZA_NAME_PROPOSE          "propose"
 #define STANZA_NAME_REPORT           "report"

src/xmpp/xmpp.h

@@ -64,6 +64,7 @@
 #define XMPP_FEATURE_USER_AVATAR_METADATA_NOTIFY "urn:xmpp:avatar:metadata+notify"
 #define XMPP_FEATURE_LAST_MESSAGE_CORRECTION     "urn:xmpp:message-correct:0"
 #define XMPP_FEATURE_MAM2                        "urn:xmpp:mam:2"
+#define XMPP_FEATURE_MAM2_EXTENDED               "urn:xmpp:mam:2#extended"
 #define XMPP_FEATURE_SPAM_REPORTING              "urn:xmpp:reporting:1"
 
 typedef enum {
@@ -260,7 +261,8 @@ void iq_autoping_check(void);
 void iq_http_upload_request(HTTPUpload* upload);
 void iq_command_list(const char* const target);
 void iq_command_exec(const char* const target, const char* const command);
-void iq_mam_request(ProfChatWin* win);
+void iq_mam_request(ProfChatWin* win, GDateTime* enddate);
+void iq_mam_request_older(ProfChatWin* win);
 void iq_register_change_password(const char* const user, const char* const password);
 void iq_muc_register_nick(const char* const roomjid);