Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable file compression on file rotation #694

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open

Enable file compression on file rotation #694

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
0c23e92
Enable gzip compression on file rotation
averater Oct 28, 2024
ce4dc32
Enable file gzip config
averater Oct 28, 2024
2808f00
Fix comment
averater Oct 28, 2024
4bcd6cc
Put compression functions in ifdefs
averater Nov 4, 2024
c65afc3
Updates from internal review
averater Nov 6, 2024
b134687
Merge branch 'master' into enable_file_compression
averater Nov 6, 2024
f8cc172
Merge branch 'COVESA:enable_file_compression' into enable_file_compre…
averater Nov 27, 2024
8253e26
Check gzlog in dlt_logstorage_sync_to_file
erikstenlund Nov 27, 2024
2478992
Fix file length calculation
averater Oct 28, 2024
c860d92
typo
averater Oct 28, 2024
8e18bcd
Make file suffix dependent on gzip compression
averater Nov 21, 2024
7229988
Merge branch 'master' into enable_file_compression
averater Dec 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/dlt_offline_logstorage.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ NOFiles=<number of files> # Number of created files before oldest is
SyncBehavior=<strategy> # Specify sync strategy. Default: Sync'ed after every message. See Logstorage Ringbuffer Implementation below.
EcuID=<ECUid> # Specify ECU identifier
SpecificSize=<spec size in bytes> # Store logs in storage devices after specific size is reached.
GzipCompression=<ON/OFF> # Write the logfiles with gzip compression.
GzipCompression=<ON/FILE/OFF> # Write the logfiles with gzip compression. ON: Continously, FILE: On file rotation
OverwriteBehavior=<strategy> # Specify overwrite strategy. Default: Delete oldest file and continue. See Logstorage Ringbuffer Implementation below.
DisableNetwork=<ON/OFF> # Specify if the message shall be routed to network client.
```
Expand Down
16 changes: 10 additions & 6 deletions src/offlinelogstorage/dlt_offline_logstorage.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1337,17 +1337,21 @@ DLT_STATIC int dlt_logstorage_check_gzip_compression(DltLogStorageFilterConfig *

if (strcasestr(value, "ON") != NULL) {
config->gzip_compression = DLT_LOGSTORAGE_GZIP_ON;
} else if (strcasestr(value, "OFF") != NULL) {
config->gzip_compression = DLT_LOGSTORAGE_GZIP_OFF;
} else {
dlt_log(LOG_WARNING,
"Unknown gzip compression flag. Set default OFF\n");
}
else if (strcasestr(value, "FILE") != NULL) {
config->gzip_compression = DLT_LOGSTORAGE_GZIP_FILE;
}
else if (strcasestr(value, "OFF") != NULL) {
config->gzip_compression = DLT_LOGSTORAGE_GZIP_OFF;
}
else {
dlt_log(LOG_WARNING, "Unknown gzip compression flag\n");
config->gzip_compression = DLT_LOGSTORAGE_GZIP_ERROR;
return 1;
}
#else
dlt_log(LOG_WARNING, "dlt-daemon not compiled with logstorage gzip support\n");
config->gzip_compression = 0;
config->gzip_compression = DLT_LOGSTORAGE_GZIP_OFF;
#endif
return 0;
}
Expand Down
9 changes: 6 additions & 3 deletions src/offlinelogstorage/dlt_offline_logstorage.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,11 +123,14 @@
#define DLT_LOGSTORAGE_DISABLE_NW_OFF 1 /* default, enable network routing */
#define DLT_LOGSTORAGE_DISABLE_NW_ON (1 << 1) /* disable network routing */

/* Offline Logstorage disable network routing */
/* Offline Logstorage enable gzip compression */
#define DLT_LOGSTORAGE_GZIP_ERROR -1 /* error case */
#define DLT_LOGSTORAGE_GZIP_UNSET 0 /* not set */
#define DLT_LOGSTORAGE_GZIP_OFF 1 /* default, enable network routing */
#define DLT_LOGSTORAGE_GZIP_ON (1 << 1) /* disable network routing */
#define DLT_LOGSTORAGE_GZIP_OFF 1 /* default, no compression */
#define DLT_LOGSTORAGE_GZIP_ON \
(1 << 1) /* enable gzip compression of all files */
#define DLT_LOGSTORAGE_GZIP_FILE \
(1 << 2) /* enable gzip compression on file rotation */

/* logstorage max cache */
extern unsigned int g_logstorage_cache_max;
Expand Down
175 changes: 158 additions & 17 deletions src/offlinelogstorage/dlt_offline_logstorage_behavior.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ void dlt_logstorage_log_file_name(char *log_file_name,
}

dlt_logstorage_concat_logfile_name(log_file_name, ".dlt");
if (filter_config->gzip_compression) {
if (filter_config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) {
dlt_logstorage_concat_logfile_name(log_file_name, ".gz");
}
}
Expand Down Expand Up @@ -370,7 +370,8 @@ int dlt_logstorage_storage_dir_info(DltLogStorageUserConfig *file_config,
config->records = NULL;
}

char *suffix = config->gzip_compression ? ".dlt.gz" : ".dlt";
char *suffix = ".dlt.gz";
char *gzsuffix = ".dlt";

for (i = 0; i < cnt; i++) {
int len = 0;
Expand All @@ -383,10 +384,13 @@ int dlt_logstorage_storage_dir_info(DltLogStorageUserConfig *file_config,
if (config->num_files == 1 && file_config->logfile_optional_counter) {
/* <filename>.dlt or <filename>_<tmsp>.dlt */
if ((files[i]->d_name[len] == suffix[0]) ||
(files[i]->d_name[len] == gzsuffix[0]) ||
(file_config->logfile_timestamp &&
(files[i]->d_name[len] == file_config->logfile_delimiter))) {
(files[i]->d_name[len] ==
file_config->logfile_delimiter))) {
current_idx = 1;
} else {
}
averater marked this conversation as resolved.
Show resolved Hide resolved
else {
continue;
}
} else {
Expand Down Expand Up @@ -486,12 +490,13 @@ DLT_STATIC void dlt_logstorage_open_log_output_file(DltLogStorageFilterConfig *c
return;
}
config->fd = fileno(file);
if (config->gzip_compression) {
if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) {
#ifdef DLT_LOGSTORAGE_USE_GZIP
dlt_vlog(LOG_DEBUG, "%s: Opening GZIP log file\n", __func__);
config->gzlog = gzdopen(config->fd, mode);
#endif
} else {
}
else {
dlt_vlog(LOG_DEBUG, "%s: Opening log file\n", __func__);
config->log = file;
}
Expand Down Expand Up @@ -729,6 +734,29 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config,
}
}

#ifdef DLT_LOGSTORAGE_USE_GZIP
if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_FILE) {
tmp = &config->records;
while ((*tmp)->next != NULL) {
int len = strlen((*tmp)->name);
const char *suffix = ".dlt";
Copy link
Collaborator

@minminlittleshrimp minminlittleshrimp Dec 6, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I need more context/info/description about this feature:

  1. Is it trying to do a dynamic activate compress mode, e.g. logging .dlt and then immediately switch to .dlt.gz using compress function for file/fd? If not, we just be fine with .gz file count from 0 or 1 since .gz and .dlt are already different file types.
  2. If it does so, why dont we just apply compress function in native logstorage file rotation, adding new func here seems doing a duplicated task?
  3. Minor suggestion: adding macro name FILE cannot directly inform user/dev that it is for file rotation, any better naming scheme here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. The feature it to log normal dlt and only when a file is "full" it compress it. This way the stored files are compressed but the current file is uncompressed. This is for the use case of copying stored files while logging is in progress.
    I think the purpose of removing older files and having a max number of files is to not fill storage. Then it would make sense to bunch dlt and gz.dlt together into one count.

  2. It is quite different tasks to compress into an open file and to compress an entire file. I thought this way was the easiest and cleanest way.

  3. I thought of it but couldn't figure out a better name that is nice and short. Maybe ARCHIVE is better?

if (strcmp(&(*tmp)->name[len - strlen(suffix)], suffix) ==
0) {
memset(absolute_file_path, 0,
sizeof(absolute_file_path) / sizeof(char));
strcat(absolute_file_path, storage_path);
strncat(absolute_file_path, (*tmp)->name, len);
dlt_vlog(LOG_INFO,
"%s: Compressing '%s' (num_log_files: %d, "
"file_name:%s)\n",
__func__, absolute_file_path, num_log_files,
(*tmp)->name);
dlt_logstorage_compress_dlt_file(absolute_file_path);
}
tmp = &(*tmp)->next;
}
}
#endif
}
}

Expand Down Expand Up @@ -758,6 +786,114 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config,
return ret;
}

#ifdef DLT_LOGSTORAGE_USE_GZIP
/**
* dlt_logstorage_compress_dlt_file
*
* Compress content of a file and remove the original file.
* compressed file name is the same as the original with .gz extension.
*
* @param file_path The file to compress
* @return 0 on success, -1 on error
*/
int dlt_logstorage_compress_dlt_file(char *file_path)
{
char file_path_dest[DLT_OFFLINE_LOGSTORAGE_MAX_PATH_LEN + 1] = {'\0'};
strcat(file_path_dest, file_path);
strcat(file_path_dest, ".gz");

FILE *source = fopen(file_path, "rb");
if (source == NULL) {
dlt_vlog(LOG_ERR, "%s: could not open %s\n", __func__, file_path);
return -1;
}

FILE *dest = fopen(file_path_dest, "wb");
if (dest == NULL) {
dlt_vlog(LOG_ERR, "%s: could not open %s\n", __func__, file_path_dest);
fclose(source);
return -1;
}

gzFile gzfile = gzdopen(fileno(dest), "wb");
if (dest == NULL) {
dlt_vlog(LOG_ERR, "%s: could not gz open %s\n", __func__,
file_path_dest);
fclose(source);
fclose(dest);
return -1;
}

int ret = dlt_logstorage_compress_fd(source, gzfile);

fclose(source);
gzclose(gzfile);
fclose(dest);

if (ret == 0 && remove(file_path) != 0) {
dlt_vlog(LOG_ERR, "%s: could not remove original file %s \n", __func__,
file_path);
return -1;
}

return ret;
}
#endif

#ifdef DLT_LOGSTORAGE_USE_GZIP
/**
* dlt_logstorage_compress_fd
*
* Compress content of a file descriptor into a gzFile.
*
* @param source The file to compress
* @param dest The file to write to
* @return 0 on success, -1 on error
*/
int dlt_logstorage_compress_fd(FILE *source, gzFile dest)
{
int ret, flush;
z_stream strm;
unsigned char in[COMPRESS_CHUNK];

/* allocate deflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
ret = deflateInit(&strm, Z_DEFAULT_COMPRESSION);
if (ret != Z_OK)
return ret;

/* compress until end of file */
do {
strm.avail_in = fread(in, 1, COMPRESS_CHUNK, source);
if (ferror(source)) {
(void)deflateEnd(&strm);
dlt_vlog(LOG_ERR, "%s: Can't open source\n", __func__);
return -1;
}

ret = gzwrite(dest, in, strm.avail_in);
if (ret == 0) {
(void)deflateEnd(&strm);
dlt_vlog(LOG_ERR, "%s: failed to write to log file\n", __func__);
return -1;
}
if (gzflush(dest, Z_SYNC_FLUSH) != 0)
dlt_vlog(LOG_ERR, "%s: failed to gzflush log file\n", __func__);

flush = feof(source) ? Z_FINISH : Z_NO_FLUSH;
strm.next_in = in;

// done when last data in file processed
} while (flush != Z_FINISH);

// clean up and return
(void)deflateEnd(&strm);
return 0;
}
#endif

/**
* dlt_logstorage_find_dlt_header
*
Expand Down Expand Up @@ -824,9 +960,10 @@ DLT_STATIC int dlt_logstorage_write_to_log(void *ptr, size_t size, size_t nmemb,
DltLogStorageFilterConfig *config)
{
#ifdef DLT_LOGSTORAGE_USE_GZIP
if (config->gzip_compression) {
if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) {
return gzfwrite(ptr, size, nmemb, config->gzlog);
} else {
}
else {
return fwrite(ptr, size, nmemb, config->log);
}
#else
Expand All @@ -851,26 +988,28 @@ DLT_STATIC void dlt_logstorage_check_write_ret(DltLogStorageFilterConfig *config
}

if (ret <= 0) {
if (config->gzip_compression) {
if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) {
#ifdef DLT_LOGSTORAGE_USE_GZIP
const char *msg = gzerror(config->gzlog, &ret);
if (msg != NULL) {
dlt_vlog(LOG_ERR, "%s: failed to write cache into log file: %s\n", __func__, msg);
}
#endif
} else {
}
else {
if (ferror(config->log) != 0)
dlt_vlog(LOG_ERR, "%s: failed to write cache into log file\n", __func__);
}
}
else {
/* force sync */
if (config->gzip_compression) {
if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) {
#ifdef DLT_LOGSTORAGE_USE_GZIP
if (gzflush(config->gzlog, Z_SYNC_FLUSH) != 0)
dlt_vlog(LOG_ERR, "%s: failed to gzflush log file\n", __func__);
#endif
} else {
}
else {
if (fflush(config->log) != 0)
dlt_vlog(LOG_ERR, "%s: failed to flush log file\n", __func__);
}
Expand Down Expand Up @@ -1097,7 +1236,7 @@ int dlt_logstorage_prepare_on_msg(DltLogStorageFilterConfig *config,
/* Sync only if on_msg */
if ((config->sync == DLT_LOGSTORAGE_SYNC_ON_MSG) ||
(config->sync == DLT_LOGSTORAGE_SYNC_UNSET)) {
if (config->gzip_compression) {
if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) {
if (fsync(fileno(config->gzlog)) != 0) {
if (errno != ENOSYS) {
dlt_vlog(LOG_ERR, "%s: failed to sync gzip log file\n", __func__);
Expand Down Expand Up @@ -1193,10 +1332,11 @@ int dlt_logstorage_write_on_msg(DltLogStorageFilterConfig *config,
dlt_log(LOG_WARNING, "Wrote less data than specified\n");

#ifdef DLT_LOGSTORAGE_USE_GZIP
if (config->gzip_compression) {
if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) {
gzerror(config->gzlog, &ret);
return ret;
} else {
}
else {
return ferror(config->log);
}
#else
Expand Down Expand Up @@ -1227,12 +1367,13 @@ int dlt_logstorage_sync_on_msg(DltLogStorageFilterConfig *config,
return -1;

if (status == DLT_LOGSTORAGE_SYNC_ON_MSG) { /* sync on every message */
if (config->gzip_compression) {
if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) {
#ifdef DLT_LOGSTORAGE_USE_GZIP
if (gzflush(config->gzlog, Z_SYNC_FLUSH) != 0)
dlt_vlog(LOG_ERR, "%s: failed to gzflush log file\n", __func__);
#endif
} else {
}
else {
if (fflush(config->log) != 0)
dlt_vlog(LOG_ERR, "%s: failed to flush log file\n", __func__);
}
Expand Down
6 changes: 6 additions & 0 deletions src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config,
bool is_update_required,
bool is_sync);

#ifdef DLT_LOGSTORAGE_USE_GZIP
#define COMPRESS_CHUNK 16384
int dlt_logstorage_compress_dlt_file(char *path);
int dlt_logstorage_compress_fd(FILE *source, gzFile dest);
#endif

DLT_STATIC int dlt_logstorage_sync_to_file(DltLogStorageFilterConfig *config,
DltLogStorageUserConfig *file_config,
char *dev_path,
Expand Down
Loading