memory leak

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
4 messages Options
Reply | Threaded
Open this post in threaded view
|

memory leak

Levente Kovacs
I'm fighting with some memory leak. From time to time the vsize of the
process goes up with 100k. I have several function like this, and I
call them in each second. My memory growth happens every 10 minutes or
so.



int dbSqliteSelectSensorsToReport(sdmd_t *sdmd, sensor_t *sensor, int
deviceType, int target) { char sqlQuery[256];
        char *tableName;
        char *targetName;
        int haveWork = 1;
        int ret, retVal;
        sqlite3_stmt *sq3Stmt;


        /*Reset the state whatever it is*/

        sensor->stat = SENSOR_DATA_INVALID;

        do {

                switch (deviceType) {
                        case SENSOR_TYPE_AN:
                        tableName = "analog";
                        break;
                        case SENSOR_TYPE_DIG:
                        tableName = "digital";
                        break;
                        case SENSOR_TYPE_UNKNOWN:
                        default:
                        tableName = NULL;
                        break;
                }


                switch (target) {
                        case TARGET_MQTT:
                        targetName = "read_mqtt";
                        break;
                        case TARGET_MODBUS: /*This makes no sence,
                hence it is done in an other process... but whatever*/
                targetName = NULL; break;
                        default:
                        targetName = NULL;
                        break;
                }

                if (targetName == NULL || tableName == NULL) {
                        retVal = DB_SQLITE_ERR;
                        break;
                }

                sprintf(sqlQuery, "SELECT rowid, sensor_id, value, time
                FROM %s WHERE %s=0 LIMIT 1;", tableName,
                targetName); /*Read one by one. I don't want to block
                DB access.*/

                sqlite3_prepare_v2(sdmd->db, sqlQuery, -1, &sq3Stmt,
                NULL);

                while (haveWork == 1) {
                        ret = sqlite3_step(sq3Stmt);
                        switch (ret) {
                                case SQLITE_ROW:
                                /*Read the data*/
                                sensor->id =
                sqlite3_column_int64(sq3Stmt, 0); sensor->sensorId =
                sqlite3_column_int64(sq3Stmt, 1); sensor->value =
                sqlite3_column_double(sq3Stmt, 2); sensor->time =
                sqlite3_column_int64(sq3Stmt, 3); sensor->stat =
                SENSOR_DATA_VALID; sprintf(sensor->name, "%s_%d",
                tableName, sensor->sensorId); break;
                                case SQLITE_DONE:
                                haveWork = 0;
                                retVal = DB_SQLITE_OK;
                                break;
                                case SQLITE_BUSY:
                                debug(sdmd->dbg, DEBUG_ERR, "%s():
                sqlite (BUSY): %s\n", __func__,
                sqlite3_errmsg(sdmd->db)); haveWork = 0; retVal =
                DB_SQLITE_ERR; break;
                                default:
                                haveWork = 0;
                                debug(sdmd->dbg, DEBUG_ERR, "%s():
                sqlite (%d): %s\n", __func__, ret,
                sqlite3_errmsg(sdmd->db)); debug(sdmd->dbg, DEBUG_ERR,
                "%s(): sqlite query: %s\n", __func__, sqlQuery); retVal
                = DB_SQLITE_ERR; break; }
                }
        } while(0);

        sqlite3_finalize(sq3Stmt); /*Release the dB*/

        return retVal;
}


What do I do wrong?

Any help is appreciated.
Levente
_______________________________________________
sqlite-users mailing list
[hidden email]
http://mailinglists.sqlite.org/cgi-bin/mailman/listinfo/sqlite-users
Reply | Threaded
Open this post in threaded view
|

Re: memory leak

Yuriy M. Kaminskiy

On 11/05/17 03:28 , Lev wrote:

> I'm fighting with some memory leak. From time to time the vsize of the
> process goes up with 100k. I have several function like this, and I
> call them in each second. My memory growth happens every 10 minutes or
> so.
>
> int dbSqliteSelectSensorsToReport(sdmd_t *sdmd, sensor_t *sensor, int
> deviceType, int target) { char sqlQuery[256];
> char *tableName;
> char *targetName;
> int haveWork = 1;
> int ret, retVal;
> sqlite3_stmt *sq3Stmt;

+       sq3Stmt = NULL;

> /*Reset the state whatever it is*/
>
> sensor->stat = SENSOR_DATA_INVALID;
>
> do {
>
> switch (deviceType) {
> case SENSOR_TYPE_AN:
> tableName = "analog";
> break;
> case SENSOR_TYPE_DIG:
> tableName = "digital";
> break;
> case SENSOR_TYPE_UNKNOWN:
> default:
> tableName = NULL;
> break;
> }
>
>
> switch (target) {
> case TARGET_MQTT:
> targetName = "read_mqtt";
> break;
> case TARGET_MODBUS: /*This makes no sence,
> hence it is done in an other process... but whatever*/
> targetName = NULL; break;
> default:
> targetName = NULL;
> break;
> }
>
> if (targetName == NULL || tableName == NULL) {
> retVal = DB_SQLITE_ERR;
> break;
> }
>
> sprintf(sqlQuery, "SELECT rowid, sensor_id, value, time
> FROM %s WHERE %s=0 LIMIT 1;", tableName,
> targetName); /*Read one by one. I don't want to block
> DB access.*/
>
> sqlite3_prepare_v2(sdmd->db, sqlQuery, -1, &sq3Stmt,
> NULL);
>
> while (haveWork == 1) {
> ret = sqlite3_step(sq3Stmt);
> switch (ret) {
> case SQLITE_ROW:
> /*Read the data*/
> sensor->id =
> sqlite3_column_int64(sq3Stmt, 0); sensor->sensorId =
> sqlite3_column_int64(sq3Stmt, 1); sensor->value =
> sqlite3_column_double(sq3Stmt, 2); sensor->time =
> sqlite3_column_int64(sq3Stmt, 3); sensor->stat =
> SENSOR_DATA_VALID; sprintf(sensor->name, "%s_%d",
> tableName, sensor->sensorId); break;
> case SQLITE_DONE:
> haveWork = 0;
> retVal = DB_SQLITE_OK;
> break;
> case SQLITE_BUSY:
> debug(sdmd->dbg, DEBUG_ERR, "%s():
> sqlite (BUSY): %s\n", __func__,
> sqlite3_errmsg(sdmd->db)); haveWork = 0; retVal =
> DB_SQLITE_ERR; break;
> default:
> haveWork = 0;
> debug(sdmd->dbg, DEBUG_ERR, "%s():
> sqlite (%d): %s\n", __func__, ret,
> sqlite3_errmsg(sdmd->db)); debug(sdmd->dbg, DEBUG_ERR,
> "%s(): sqlite query: %s\n", __func__, sqlQuery); retVal
> = DB_SQLITE_ERR; break; }
> }
> } while(0);
>
> sqlite3_finalize(sq3Stmt); /*Release the dB*/
>
> return retVal;
> }
>
>
> What do I do wrong?

I don't see anything that can trigger memory leak here.

You seems don't initialize sq3Stmt (see above +), so that you will call
sqlite3_finalize on random junk from stack on some error paths; but that
should not trigger any leaks (more likely result would be heap
corruption and crash).

And while current uses are probably safe, I'd replace sprintf with
snprintf to avoid any nasty surprises.

Maybe sqlite just use some memory for page cache (IIRC, by default,
cache size is 2MB).

Also allocated vsize can increase due to memory fragmentation (depending
on chosen malloc implementation and its fine-tuning).

You can also try running with valgrind or other memory debugger.

> Any help is appreciated.

_______________________________________________
sqlite-users mailing list
[hidden email]
http://mailinglists.sqlite.org/cgi-bin/mailman/listinfo/sqlite-users
Reply | Threaded
Open this post in threaded view
|

Re: [EXTERNAL] memory leak

Hick Gunter
In reply to this post by Levente Kovacs
You are preparing statements inside the loop, but only finalizing the last one (i.e. outside the loop)

-----Urspr√ľngliche Nachricht-----
Von: sqlite-users [mailto:[hidden email]] Im Auftrag von Lev
Gesendet: Sonntag, 05. November 2017 01:28
An: [hidden email]
Betreff: [EXTERNAL] [sqlite] memory leak

I'm fighting with some memory leak. From time to time the vsize of the process goes up with 100k. I have several function like this, and I call them in each second. My memory growth happens every 10 minutes or so.



int dbSqliteSelectSensorsToReport(sdmd_t *sdmd, sensor_t *sensor, int deviceType, int target) { char sqlQuery[256];
        char *tableName;
        char *targetName;
        int haveWork = 1;
        int ret, retVal;
        sqlite3_stmt *sq3Stmt;


        /*Reset the state whatever it is*/

        sensor->stat = SENSOR_DATA_INVALID;

        do {

                switch (deviceType) {
                        case SENSOR_TYPE_AN:
                        tableName = "analog";
                        break;
                        case SENSOR_TYPE_DIG:
                        tableName = "digital";
                        break;
                        case SENSOR_TYPE_UNKNOWN:
                        default:
                        tableName = NULL;
                        break;
                }


                switch (target) {
                        case TARGET_MQTT:
                        targetName = "read_mqtt";
                        break;
                        case TARGET_MODBUS: /*This makes no sence,
                hence it is done in an other process... but whatever*/
                targetName = NULL; break;
                        default:
                        targetName = NULL;
                        break;
                }

                if (targetName == NULL || tableName == NULL) {
                        retVal = DB_SQLITE_ERR;
                        break;
                }

                sprintf(sqlQuery, "SELECT rowid, sensor_id, value, time
                FROM %s WHERE %s=0 LIMIT 1;", tableName,
                targetName); /*Read one by one. I don't want to block
                DB access.*/

                sqlite3_prepare_v2(sdmd->db, sqlQuery, -1, &sq3Stmt,
                NULL);

                while (haveWork == 1) {
                        ret = sqlite3_step(sq3Stmt);
                        switch (ret) {
                                case SQLITE_ROW:
                                /*Read the data*/
                                sensor->id =
                sqlite3_column_int64(sq3Stmt, 0); sensor->sensorId =
                sqlite3_column_int64(sq3Stmt, 1); sensor->value =
                sqlite3_column_double(sq3Stmt, 2); sensor->time =
                sqlite3_column_int64(sq3Stmt, 3); sensor->stat =
                SENSOR_DATA_VALID; sprintf(sensor->name, "%s_%d",
                tableName, sensor->sensorId); break;
                                case SQLITE_DONE:
                                haveWork = 0;
                                retVal = DB_SQLITE_OK;
                                break;
                                case SQLITE_BUSY:
                                debug(sdmd->dbg, DEBUG_ERR, "%s():
                sqlite (BUSY): %s\n", __func__,
                sqlite3_errmsg(sdmd->db)); haveWork = 0; retVal =
                DB_SQLITE_ERR; break;
                                default:
                                haveWork = 0;
                                debug(sdmd->dbg, DEBUG_ERR, "%s():
                sqlite (%d): %s\n", __func__, ret,
                sqlite3_errmsg(sdmd->db)); debug(sdmd->dbg, DEBUG_ERR,
                "%s(): sqlite query: %s\n", __func__, sqlQuery); retVal
                = DB_SQLITE_ERR; break; }
                }
        } while(0);

        sqlite3_finalize(sq3Stmt); /*Release the dB*/

        return retVal;
}


What do I do wrong?

Any help is appreciated.
Levente
_______________________________________________
sqlite-users mailing list
[hidden email]
http://mailinglists.sqlite.org/cgi-bin/mailman/listinfo/sqlite-users


___________________________________________
 Gunter Hick | Software Engineer | Scientific Games International GmbH | Klitschgasse 2-4, A-1130 Vienna | FN 157284 a, HG Wien, DVR: 0430013 | (O) +43 1 80100 - 0

May be privileged. May be confidential. Please delete if not the addressee.
_______________________________________________
sqlite-users mailing list
[hidden email]
http://mailinglists.sqlite.org/cgi-bin/mailman/listinfo/sqlite-users
Reply | Threaded
Open this post in threaded view
|

Re: [EXTERNAL] memory leak

Yuriy M. Kaminskiy
On 11/06/17 10:02 , Hick Gunter wrote:
> You are preparing statements inside the loop, but only finalizing the
> last one (i.e. outside the loop)

This is not a real loop, it executes one time only (do {...} while(0)).

> -----Urspr√ľngliche Nachricht-----
> Von: sqlite-users [mailto:[hidden email]] Im Auftrag von Lev
> Gesendet: Sonntag, 05. November 2017 01:28
> An: [hidden email]
> Betreff: [EXTERNAL] [sqlite] memory leak
>
> I'm fighting with some memory leak. From time to time the vsize of the process goes up with 100k. I have several function like this, and I call them in each second. My memory growth happens every 10 minutes or so.
>         do {
>
...
>
>                 sqlite3_prepare_v2(sdmd->db, sqlQuery, -1, &sq3Stmt,
>                 NULL);
...
>         } while(0);
>
>         sqlite3_finalize(sq3Stmt); /*Release the dB*/

_______________________________________________
sqlite-users mailing list
[hidden email]
http://mailinglists.sqlite.org/cgi-bin/mailman/listinfo/sqlite-users