diff --git a/res/css/views/emojipicker/_EmojiPicker.scss b/res/css/views/emojipicker/_EmojiPicker.scss
index 99a75b9d10..50eeb4281c 100644
--- a/res/css/views/emojipicker/_EmojiPicker.scss
+++ b/res/css/views/emojipicker/_EmojiPicker.scss
@@ -49,7 +49,11 @@ limitations under the License.
         fill: $primary-fg-color;
     }
 
-    &:hover {
+    &:disabled svg {
+        fill: $focus-bg-color;
+    }
+
+    &:not(:disabled):hover {
         background-color: $focus-bg-color;
         border-bottom: 2px solid $button-bg-color;
     }
@@ -73,11 +77,17 @@ limitations under the License.
         border-radius: 4px 0;
     }
 
-    svg {
-        align-self: center;
-        width: 16px;
-        height: 16px;
+    button {
+        border: none;
+        background-color: inherit;
+        padding: 0;
         margin: 8px;
+
+        svg {
+            align-self: center;
+            width: 16px;
+            height: 16px;
+        }
     }
 }
 
diff --git a/src/components/views/emojipicker/Category.js b/src/components/views/emojipicker/Category.js
index 9573a24630..629dd3e570 100644
--- a/src/components/views/emojipicker/Category.js
+++ b/src/components/views/emojipicker/Category.js
@@ -23,31 +23,27 @@ class Category extends React.PureComponent {
     static propTypes = {
         emojis: PropTypes.arrayOf(PropTypes.object).isRequired,
         name: PropTypes.string.isRequired,
+        id: PropTypes.string.isRequired,
         onMouseEnter: PropTypes.func.isRequired,
         onMouseLeave: PropTypes.func.isRequired,
         onClick: PropTypes.func.isRequired,
-        filter: PropTypes.string,
     };
 
     render() {
         const { onClick, onMouseEnter, onMouseLeave, emojis, name, filter } = this.props;
-
-        const Emoji = sdk.getComponent("emojipicker.Emoji");
-        const renderedEmojis = (emojis || []).map(emoji => !filter || emoji.filterString.includes(filter) ? (
-            <Emoji key={emoji.hexcode} emoji={emoji}
-                onClick={onClick} onMouseEnter={onMouseEnter} onMouseLeave={onMouseLeave} />
-        ) : null).filter(component => component !== null);
-        if (renderedEmojis.length === 0) {
+        if (!emojis || emojis.length === 0) {
             return null;
         }
 
+        const Emoji = sdk.getComponent("emojipicker.Emoji");
         return (
-            <section className="mx_EmojiPicker_category">
+            <section className="mx_EmojiPicker_category" data-category-id={this.props.id}>
                 <h2 className="mx_EmojiPicker_category_label">
                     {name}
                 </h2>
                 <ul className="mx_EmojiPicker_list">
-                    {renderedEmojis}
+                    {emojis.map(emoji => <Emoji key={emoji.hexcode} emoji={emoji}
+                        onClick={onClick} onMouseEnter={onMouseEnter} onMouseLeave={onMouseLeave} />)}
                 </ul>
             </section>
         )
diff --git a/src/components/views/emojipicker/EmojiPicker.js b/src/components/views/emojipicker/EmojiPicker.js
index 0ffe3a06f7..d1f784f062 100644
--- a/src/components/views/emojipicker/EmojiPicker.js
+++ b/src/components/views/emojipicker/EmojiPicker.js
@@ -16,11 +16,13 @@ limitations under the License.
 
 import React from 'react';
 import PropTypes from 'prop-types';
-
 import EMOJIBASE from 'emojibase-data/en/compact.json';
+
 import sdk from '../../../index';
 import { _t } from '../../../languageHandler';
 
+import * as recent from './recent';
+
 const EMOJIBASE_CATEGORY_IDS = [
     "people", // smileys
     "people", // actually people
@@ -43,11 +45,15 @@ const DATA_BY_CATEGORY = {
     "objects": [],
     "symbols": [],
     "flags": [],
-    "control": [],
 };
+const DATA_BY_EMOJI = {};
 
 EMOJIBASE.forEach(emoji => {
-    DATA_BY_CATEGORY[EMOJIBASE_CATEGORY_IDS[emoji.group]].push(emoji);
+    DATA_BY_EMOJI[emoji.unicode] = emoji;
+    const categoryId = EMOJIBASE_CATEGORY_IDS[emoji.group];
+    if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) {
+        DATA_BY_CATEGORY[categoryId].push(emoji);
+    }
     // This is used as the string to match the query against when filtering emojis.
     emoji.filterString = `${emoji.annotation}\n${emoji.shortcodes.join('\n')}}\n${emoji.emoticon || ''}`;
 });
@@ -65,49 +71,76 @@ class EmojiPicker extends React.Component {
             previewEmoji: null,
         };
 
+        this.bodyRef = React.createRef();
+
+        this.recentlyUsed = recent.get().map(unicode => DATA_BY_EMOJI[unicode]);
+        this.memoizedDataByCategory = {
+            recent: this.recentlyUsed,
+            ...DATA_BY_CATEGORY,
+        };
+
         this.categories = [{
             id: "recent",
             name: _t("Frequently Used"),
+            enabled: this.recentlyUsed.length > 0,
         }, {
             id: "people",
             name: _t("Smileys & People"),
+            enabled: true,
         }, {
             id: "nature",
             name: _t("Animals & Nature"),
+            enabled: true,
         }, {
             id: "foods",
             name: _t("Food & Drink"),
+            enabled: true,
         }, {
             id: "activity",
             name: _t("Activities"),
+            enabled: true,
         }, {
             id: "places",
             name: _t("Travel & Places"),
+            enabled: true,
         }, {
             id: "objects",
             name: _t("Objects"),
+            enabled: true,
         }, {
             id: "symbols",
             name: _t("Symbols"),
+            enabled: true,
         }, {
             id: "flags",
             name: _t("Flags"),
+            enabled: true,
         }];
 
         this.onChangeFilter = this.onChangeFilter.bind(this);
         this.onHoverEmoji = this.onHoverEmoji.bind(this);
         this.onHoverEmojiEnd = this.onHoverEmojiEnd.bind(this);
         this.onClickEmoji = this.onClickEmoji.bind(this);
+        this.scrollToCategory = this.scrollToCategory.bind(this);
+
+        window.bodyRef = this.bodyRef;
     }
 
-    scrollToCategory() {
-        // TODO
+    scrollToCategory(category) {
+        const index = this.categories.findIndex(cat => cat.id === category);
+        this.bodyRef.current.querySelector(`[data-category-id="${category}"]`).scrollIntoView();
     }
 
-    onChangeFilter(ev) {
-        this.setState({
-            filter: ev.target.value,
-        });
+    onChangeFilter(filter) {
+        for (let [id, emojis] of Object.entries(this.memoizedDataByCategory)) {
+            // If the new filter string includes the old filter string, we don't have to re-filter the whole dataset.
+            if (!filter.includes(this.state.filter)) {
+                emojis = id === "recent" ? this.recentlyUsed : DATA_BY_CATEGORY[id];
+            }
+            this.memoizedDataByCategory[id] = emojis.filter(emoji => emoji.filterString.includes(filter));
+            this.categories.find(cat => cat.id === id).enabled = this.memoizedDataByCategory[id].length > 0;
+        }
+        this.setState({ filter });
     }
 
     onHoverEmoji(emoji) {
@@ -124,6 +157,10 @@ class EmojiPicker extends React.Component {
 
     onClickEmoji(emoji) {
         this.props.onChoose(emoji.unicode);
+        recent.add(emoji.unicode);
+        this.recentlyUsed = recent.get().map(unicode => DATA_BY_EMOJI[unicode]);
+        this.memoizedDataByCategory.recent = this.recentlyUsed.filter(emoji =>
+            emoji.filterString.includes(this.state.filter))
     }
 
     render() {
@@ -136,10 +173,10 @@ class EmojiPicker extends React.Component {
             <div className="mx_EmojiPicker">
                 <Header categories={this.categories} defaultCategory="recent" onAnchorClick={this.scrollToCategory}/>
                 <Search query={this.state.filter} onChange={this.onChangeFilter}/>
-                <div className="mx_EmojiPicker_body">
+                <div className="mx_EmojiPicker_body" ref={this.bodyRef}>
                     {this.categories.map(category => (
-                        <Category key={category.id} emojis={DATA_BY_CATEGORY[category.id]} name={category.name}
-                            filter={this.state.filter} onClick={this.onClickEmoji}
+                        <Category key={category.id} id={category.id} name={category.name}
+                            emojis={this.memoizedDataByCategory[category.id]} onClick={this.onClickEmoji}
                             onMouseEnter={this.onHoverEmoji} onMouseLeave={this.onHoverEmojiEnd} />
                     ))}
                 </div>
diff --git a/src/components/views/emojipicker/Header.js b/src/components/views/emojipicker/Header.js
index d061f8559a..95af68f4a6 100644
--- a/src/components/views/emojipicker/Header.js
+++ b/src/components/views/emojipicker/Header.js
@@ -34,8 +34,7 @@ class Header extends React.Component {
         this.handleClick = this.handleClick.bind(this);
     }
 
-    handleClick(ev) {
-        const selected = ev.target.getAttribute("data-category-id");
+    handleClick(selected) {
         this.setState({selected});
         this.props.onAnchorClick(selected);
     };
@@ -44,9 +43,9 @@ class Header extends React.Component {
         return (
             <nav className="mx_EmojiPicker_header">
                 {this.props.categories.map(category => (
-                    <button key={category.id} className={`mx_EmojiPicker_anchor ${
+                    <button disabled={!category.enabled} key={category.id} className={`mx_EmojiPicker_anchor ${
                         this.state.selected === category.id ? 'mx_EmojiPicker_anchor_selected' : ''}`}
-                        onClick={this.handleClick} data-category-id={category.id} title={category.name}>
+                        onClick={() => this.handleClick(category.id)} title={category.name}>
                         {icons.categories[category.id]()}
                     </button>
                 ))}
diff --git a/src/components/views/emojipicker/Search.js b/src/components/views/emojipicker/Search.js
index a0200a29d4..cb41eefd1d 100644
--- a/src/components/views/emojipicker/Search.js
+++ b/src/components/views/emojipicker/Search.js
@@ -28,8 +28,11 @@ class Search extends React.PureComponent {
     render() {
         return (
             <div className="mx_EmojiPicker_search">
-                <input type="text" placeholder="Search" value={this.props.query} onChange={this.props.onChange}/>
-                {icons.search.search()}
+                <input type="text" placeholder="Search" value={this.props.query}
+                    onChange={ev => this.props.onChange(ev.target.value)}/>
+                <button onClick={() => this.props.onChange("")}>
+                    {this.props.query ? icons.search.delete() : icons.search.search()}
+                </button>
             </div>
         )
     }
diff --git a/src/components/views/emojipicker/recent.js b/src/components/views/emojipicker/recent.js
new file mode 100644
index 0000000000..f37fbdb235
--- /dev/null
+++ b/src/components/views/emojipicker/recent.js
@@ -0,0 +1,35 @@
+/*
+Copyright 2019 Tulir Asokan
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const REACTION_COUNT = JSON.parse(window.localStorage.mx_reaction_count || '{}');
+let sorted = null;
+
+export function add(emoji) {
+    const [count] = REACTION_COUNT[emoji] || [0];
+    REACTION_COUNT[emoji] = [count + 1, Date.now()];
+    window.localStorage.mx_reaction_count = JSON.stringify(REACTION_COUNT);
+    sorted = null;
+}
+
+export function get(limit = 24) {
+    if (sorted === null) {
+        sorted = Object.entries(REACTION_COUNT)
+            .sort(([, [count1, date1]], [, [count2, date2]]) =>
+                count2 === count1 ? date2 - date1 : count2 - count1)
+            .map(([emoji, count]) => emoji);
+    }
+    return sorted.slice(0, limit);
+}
diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json
index 44812f8bbc..4b4330cf7a 100644
--- a/src/i18n/strings/en_EN.json
+++ b/src/i18n/strings/en_EN.json
@@ -1839,5 +1839,6 @@
     "Travel & Places": "Travel & Places",
     "Objects": "Objects",
     "Symbols": "Symbols",
-    "Flags": "Flags"
+    "Flags": "Flags",
+    "React": "React"
 }