Skip to content

Commit de703e0

Browse files
siara-ccsiara-in
siara-cc
authored and
siara-in
committed
include Web console and Unishox
1 parent 800bf5e commit de703e0

File tree

8 files changed

+1558
-6
lines changed

8 files changed

+1558
-6
lines changed

‎README.md

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ Any Flash memory such as those available on SPIFFS or Micro SD cards have limita
104104

105105
## Compression with Shox96
106106

107+
(Shox96 is deprecated and Unishox explained below will be supported in future).
108+
107109
This implementation of `sqlite3` includes two functions `shox96_0_2c()` and `shox96_0_2d()` for compressing and decompressing text data.
108110

109111
Shox96 is a compression technique developed for reducing storage size of Short Strings. Details of how it works can be found [here](https://github.com/siara-cc/Shox96).
@@ -132,6 +134,24 @@ See screenshots section for output.
132134
- It does not work if the string has binary characters. that is, other than ASCII 32 to 126, CR, LF and Tab.
133135
- Dictionary based compression / decompression is not yet implemented.
134136

137+
## Compression with Unishox
138+
139+
This implementation also includes two functions `unishox1c()` and `unishox1d()` for compressing and decompressing text data.
140+
141+
Unishox is a compression technique developed for reducing storage size of Short Unicode Strings. Details of how it works can be found [here](https://github.com/siara-cc/Unishox).
142+
143+
In general it can achieve upto 40% size reduction for Short Strings.
144+
145+
### Usage
146+
147+
The usage is similar to that of Shox96, only in this case UTF-8 strings can be used.
148+
149+
See screenshots section for output.
150+
151+
### Limitations (for Unishox)
152+
153+
- Trying to decompress any blob that was not compressed using `unishox1c()` will crash the program.
154+
135155
## Acknowledgements
136156

137157
* This library was developed based on NodeMCU module developed by [Luiz Felipe Silva](https://github.com/luizfeliperj). The documentation can be found [here](https://nodemcu.readthedocs.io/en/master/en/modules/sqlite3/).
@@ -155,9 +175,9 @@ See screenshots section for output.
155175

156176
![](console_screenshot.png?raw=true)
157177

158-
### Shox96 compression
178+
### Unishox compression
159179

160-
![](output_shox96.png?raw=true)
180+
![](output_web_console.png?raw=true)
161181

162182
### Output of Querying StackOverflow DB through WebServer example:
163183

‎examples/sqlite3_console/sqlite3_console.ino

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -315,7 +315,7 @@ void loop() {
315315
Serial.printf("\nHeap size: %d\n", ESP.getHeapSize());
316316
Serial.printf("Free Heap: %d\n", esp_get_free_heap_size());
317317
Serial.printf("Min Free Heap: %d\n", esp_get_minimum_free_heap_size());
318-
Serial.printf("Largest Free block: %d\n", heap_caps_get_largest_free_block());
318+
Serial.printf("Largest Free block: %d\n", heap_caps_get_largest_free_block(MALLOC_CAP_8BIT));
319319
break;
320320
default:
321321
Serial.println(F("Invalid choice. Try again."));

‎examples/sqlite3_web_console/sqlite3_web_console.ino

Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
/*
2+
This utility provides ability to run Sql Statements on Sqlite3 databases from SD Card
3+
or SPIFFS through the Web Server and display in the form of HTML page.
4+
5+
For more information, visit https://github.com/siara-cc/esp32_arduino_sqlite3_lib
6+
7+
Copyright (c) 2018, Siara Logics (cc)
8+
*/
9+
10+
/*
11+
Copyright (c) 2015, Majenko Technologies
12+
All rights reserved.
13+
14+
Redistribution and use in source and binary forms, with or without modification,
15+
are permitted provided that the following conditions are met:
16+
17+
* * Redistributions of source code must retain the above copyright notice, this
18+
list of conditions and the following disclaimer.
19+
20+
* * Redistributions in binary form must reproduce the above copyright notice, this
21+
list of conditions and the following disclaimer in the documentation and/or
22+
other materials provided with the distribution.
23+
24+
* * Neither the name of Majenko Technologies nor the names of its
25+
contributors may be used to endorse or promote products derived from
26+
this software without specific prior written permission.
27+
28+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
29+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
30+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
31+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
32+
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
33+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
34+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
35+
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38+
*/
39+
40+
#include <WiFi.h>
41+
#include <WiFiClient.h>
42+
#include <WebServer.h>
43+
#include <ESPmDNS.h>
44+
#include <sqlite3.h>
45+
#include <SPI.h>
46+
#include <FS.h>
47+
#include "SPIFFS.h"
48+
#include "SD_MMC.h"
49+
#include "SD.h"
50+
51+
const char *ssid = "Nokia1";
52+
const char *password = "nokiafive";
53+
54+
WebServer server(80);
55+
56+
const int led = 13;
57+
58+
void handleRoot(const char *db_name, const char *sql) {
59+
digitalWrite ( led, 1 );
60+
String temp;
61+
int sec = millis() / 1000;
62+
int min = sec / 60;
63+
int hr = min / 60;
64+
65+
temp = "<html><head>\
66+
<title>ESP32 Sqlite Web Console</title>\
67+
<style>\
68+
body { background-color: #cccccc; font-family: Arial, Helvetica, Sans-Serif; font-size: large; Color: #000088; }\
69+
</style>\
70+
</head>\
71+
<body>\
72+
<h2>ESP32 Sqlite Web Console</h2>\
73+
<p>Uptime: ";
74+
temp += hr;
75+
temp += ":";
76+
temp += min % 60;
77+
temp += ":";
78+
temp += sec % 60;
79+
temp += "</p>\
80+
<form name='params' method='POST' action='exec_sql'>\
81+
<textarea style='font-size: medium; width:100%' rows='4' placeholder='Enter SQL Statement' name='sql'>";
82+
if (sql != NULL)
83+
temp += sql;
84+
temp += "</textarea> \
85+
<br>File name (prefix with /spiffs/ or /sd/ or /sdcard/):<br/><input type=text size='50' style='font-size: small' value='";
86+
if (db_name != NULL)
87+
temp += db_name;
88+
temp += "' name='db_name'/> \
89+
<br><br><input type=submit style='font-size: large' value='Execute'/>\
90+
</form><hr/>";
91+
92+
server.send ( 200, "text/html", temp.c_str() );
93+
digitalWrite ( led, 0 );
94+
}
95+
96+
void handleNotFound() {
97+
digitalWrite ( led, 1 );
98+
String message = "File Not Found\n\n";
99+
message += "URI: ";
100+
message += server.uri();
101+
message += "\nMethod: ";
102+
message += ( server.method() == HTTP_GET ) ? "GET" : "POST";
103+
message += "\nArguments: ";
104+
message += server.args();
105+
message += "\n";
106+
107+
for ( uint8_t i = 0; i < server.args(); i++ ) {
108+
message += " " + server.argName ( i ) + ": " + server.arg ( i ) + "\n";
109+
}
110+
111+
server.send ( 404, "text/plain", message );
112+
digitalWrite ( led, 0 );
113+
}
114+
115+
sqlite3 *db1;
116+
int rc;
117+
sqlite3_stmt *res;
118+
int rec_count = 0;
119+
const char *tail;
120+
char current_db[255];
121+
122+
int openDb(const char *filename) {
123+
if (strncmp(filename, current_db, sizeof(current_db)) == 0)
124+
return 0;
125+
else
126+
sqlite3_close(db1);
127+
int rc = sqlite3_open(filename, &db1);
128+
if (rc) {
129+
Serial.printf("Can't open database: %s\n", sqlite3_errmsg(db1));
130+
memset(current_db, '\0', sizeof(current_db));
131+
return rc;
132+
} else {
133+
Serial.printf("Opened database successfully\n");
134+
strcpy(current_db, filename);
135+
}
136+
return rc;
137+
}
138+
139+
void setup ( void ) {
140+
pinMode(led, OUTPUT);
141+
digitalWrite(led, 0);
142+
Serial.begin(115200);
143+
WiFi.mode(WIFI_STA);
144+
WiFi.begin(ssid, password);
145+
Serial.println("");
146+
147+
// Wait for connection
148+
while ( WiFi.status() != WL_CONNECTED ) {
149+
delay ( 500 );
150+
Serial.print ( "." );
151+
}
152+
153+
Serial.println ( "" );
154+
Serial.print ( "Connected to " );
155+
Serial.println ( ssid );
156+
Serial.print ( "IP address: " );
157+
Serial.println ( WiFi.localIP() );
158+
159+
if ( MDNS.begin ( "esp32" ) ) {
160+
Serial.println ( "MDNS responder started" );
161+
}
162+
163+
memset(current_db, '\0', sizeof(current_db));
164+
165+
if (!SPIFFS.begin(true)) {
166+
Serial.println(F("Failed to mount file Serial"));
167+
return;
168+
}
169+
SPI.begin();
170+
SD_MMC.begin();
171+
SD.begin();
172+
sqlite3_initialize();
173+
174+
server.on ( "/", []() {
175+
handleRoot(NULL, NULL);
176+
});
177+
server.on ( "/exec_sql", []() {
178+
String db_name = server.arg("db_name");
179+
String sql = server.arg("sql");
180+
server.setContentLength(CONTENT_LENGTH_UNKNOWN);
181+
handleRoot(db_name.c_str(), sql.c_str());
182+
if (openDb(db_name.c_str())) {
183+
String resp = "Error opening database: ";
184+
resp += sqlite3_errmsg(db1);
185+
resp += "<pre>";
186+
resp += server.arg("db_name");
187+
resp += "</pre>";
188+
resp += ".<br><br><input type=button onclick='location.href=\"/\"' value='back'/>";
189+
server.sendContent(resp);
190+
return;
191+
}
192+
//String sql = "Select count(*) from gendered_names where name between '";
193+
//sql += server.arg("from");
194+
//sql += "' and '";
195+
//sql += server.arg("to");
196+
//sql += "'";
197+
//rc = sqlite3_prepare_v2(db1, sql.c_str(), 1000, &res, &tail);
198+
//if (rc != SQLITE_OK) {
199+
// String resp = "Failed to fetch data: ";
200+
// resp += sqlite3_errmsg(db1);
201+
// resp += ".<br><br><input type=button onclick='location.href=\"/\"' value='back'/>";
202+
// server.send ( 200, "text/html", resp.c_str());
203+
// Serial.println(resp.c_str());
204+
// return;
205+
//}
206+
//while (sqlite3_step(res) == SQLITE_ROW) {
207+
// rec_count = sqlite3_column_int(res, 0);
208+
// if (rec_count > 5000) {
209+
// String resp = "Too many records: ";
210+
// resp += rec_count;
211+
// resp += ". Please select different range";
212+
// resp += ".<br><br><input type=button onclick='location.href=\"/\"' value='back'/>";
213+
// server.send ( 200, "text/html", resp.c_str());
214+
// Serial.println(resp.c_str());
215+
// sqlite3_finalize(res);
216+
// return;
217+
// }
218+
//}
219+
//sqlite3_finalize(res);
220+
221+
rc = sqlite3_prepare_v2(db1, sql.c_str(), 1000, &res, &tail);
222+
if (rc != SQLITE_OK) {
223+
String resp = "Failed to fetch data: ";
224+
resp += sqlite3_errmsg(db1);
225+
resp += "<br><br><a href='/'>back</a>";
226+
server.sendContent(resp);
227+
Serial.println(resp.c_str());
228+
return;
229+
}
230+
231+
rec_count = 0;
232+
String resp = "<h2>Result:</h2><h3>";
233+
resp += sql;
234+
resp += "</h3><table cellspacing='1' cellpadding='1' border='1'>";
235+
server.sendContent(resp);
236+
bool first = true;
237+
while (sqlite3_step(res) == SQLITE_ROW) {
238+
resp = "";
239+
if (first) {
240+
int count = sqlite3_column_count(res);
241+
if (count == 0) {
242+
resp += "<tr><td>Statement executed successfully</td></tr>";
243+
rec_count = sqlite3_changes(db1);
244+
break;
245+
}
246+
resp += "<tr>";
247+
for (int i = 0; i<count; i++) {
248+
resp += "<td>";
249+
resp += sqlite3_column_name(res, i);
250+
resp += "</td>";
251+
}
252+
resp += "</tr>";
253+
first = false;
254+
}
255+
int count = sqlite3_column_count(res);
256+
resp += "<tr>";
257+
for (int i = 0; i<count; i++) {
258+
resp += "<td>";
259+
resp += (const char *) sqlite3_column_text(res, i);
260+
resp += "</td>";
261+
}
262+
resp += "</tr>";
263+
server.sendContent(resp);
264+
rec_count++;
265+
}
266+
resp = "</table><br><br>Number of records: ";
267+
resp += rec_count;
268+
resp += ".<br><br><input type=button onclick='location.href=\"/\"' value='back'/>";
269+
server.sendContent(resp);
270+
sqlite3_finalize(res);
271+
} );
272+
server.onNotFound ( handleNotFound );
273+
server.begin();
274+
Serial.println ( "HTTP server started" );
275+
}
276+
277+
void loop ( void ) {
278+
server.handleClient();
279+
}

‎library.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name=Sqlite3Esp32
2-
version=2.0
2+
version=2.1
33
author=Arundale Ramanathan
44
maintainer=Arun <[email protected]>
55
sentence=Sqlite3 database library for ESP32 core

‎output_web_console.png

184 KB
Loading

0 commit comments

Comments
 (0)