Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add test for jinja template test #177

Merged
merged 8 commits into from
Apr 24, 2019
Merged

add test for jinja template test #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
95cadae
clean up the logic about sampleDir and jinja_template_dir setting
GordonWang Apr 23, 2019
41d89f4
revert the change about token, sample mode test cases
GordonWang Apr 23, 2019
a2980bb
merge the change from develop branch
GordonWang Apr 23, 2019
aa34db6
merge test conf from develop branch
GordonWang Apr 23, 2019
13a0c2f
use urllib 1.24.2
GordonWang Apr 23, 2019
e504541
format with pep8
GordonWang Apr 23, 2019
c537b12
fix bug in template dir resolution
GordonWang Apr 24, 2019
aa57e3e
update test case to address multiline event
GordonWang Apr 24, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def readme():
'httplib2',
'jinja2',
'pyrabbit==1.1.0',
'urllib3==1.23',
'urllib3==1.24.2',
'pyOpenSSL',
'flake8>=3.7.7',
'yapf>=0.26.0',
Expand Down
44 changes: 19 additions & 25 deletions splunk_eventgen/lib/eventgenconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ class Config(object):
"""Reads configuration from files or Splunk REST endpoint and stores them in a 'Borg' global.
Borg is a variation on the Singleton design pattern which allows us to continually instantiate
the configuration object throughout the application and maintain state."""
DEFAULT_SAMPLE_DIR = 'samples'

# Stolen from http://code.activestate.com/recipes/66531/
# This implements a Borg patterns, similar to Singleton
# It allows numerous instantiations but always shared state
Expand Down Expand Up @@ -310,6 +312,7 @@ def parse(self):
# If we see the sample in two places, use the first and ignore the second
if not sampleexists:
s = Sample(stanza)
s.splunkEmbedded = self.splunkEmbedded

s.updateConfig(self)

Expand Down Expand Up @@ -441,38 +444,29 @@ def parse(self):
self.logger.debug("Sample directory not specified in config, setting based on standard")
if self.splunkEmbedded and not STANDALONE:
s.sampleDir = os.path.normpath(
os.path.join(self.grandparentdir, '..', '..', '..', s.app, 'samples'))
os.path.join(self.grandparentdir, os.path.pardir, os.path.pardir, os.path.pardir, s.app, self.DEFAULT_SAMPLE_DIR))
else:
# 2/1/15 CS Adding support for looking for samples based on the config file specified on
# the command line.
if self.configfile:
if os.path.isdir(self.configfile):
s.sampleDir = os.path.join(self.configfile, 'samples')
else:
s.sampleDir = os.path.join(os.getcwd(), 'samples')
base_dir = os.path.dirname(self.configfile) if os.path.isdir(self.configfile) else os.path.dirname(os.path.dirname(self.configfile))
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this part is changed to get the sampleDir according to the conf file directory.

s.sampleDir = os.path.join(base_dir, self.DEFAULT_SAMPLE_DIR)
else:
s.sampleDir = os.path.join(os.getcwd(), 'samples')
if not os.path.exists(s.sampleDir):
newSampleDir = os.path.join(os.sep.join(os.getcwd().split(os.sep)[:-1]), 'samples')
self.logger.error("Path not found for samples '%s', trying '%s'" % (s.sampleDir, newSampleDir))
s.sampleDir = newSampleDir

s.sampleDir = os.path.join(os.getcwd(), self.DEFAULT_SAMPLE_DIR)
if not os.path.exists(s.sampleDir):
newSampleDir = os.path.join(self.grandparentdir, 'samples')
self.logger.error(
"Path not found for samples '%s', trying '%s'" % (s.sampleDir, newSampleDir))
s.sampleDir = newSampleDir
# use the prebuilt sample dirs as the last choice
if not os.path.exists(s.sampleDir):
newSampleDir = os.path.join(self.grandparentdir, self.DEFAULT_SAMPLE_DIR)
self.logger.error(
"Path not found for samples '%s', trying '%s'" % (s.sampleDir, newSampleDir))
s.sampleDir = newSampleDir
else:
self.logger.debug("Sample directory specified in config, checking for relative")
# Allow for relative paths to the base directory
if not os.path.exists(s.sampleDir):
temp_sampleDir = os.path.join(self.grandparentdir, s.sampleDir)
# check the greatgrandparent just incase for the sample file.
if not os.path.exists(temp_sampleDir):
temp_sampleDir = os.path.join(self.greatgrandparentdir, s.sampleDir)
s.sampleDir = temp_sampleDir
else:
s.sampleDir = s.sampleDir
if not os.path.isabs(s.sampleDir):
# relative path use the conffile dir as the base dir
self.logger.debug("Sample directory specified in config, checking for relative")
base_path = self.configfile if os.path.isdir(self.configfile) else os.path.dirname(self.configfile)
s.sampleDir = os.path.join(base_path, s.sampleDir)
# do nothing when sampleDir is absolute path

# 2/1/15 CS Adding support for command line options, specifically running a single sample
# from the command line
Expand Down
1 change: 1 addition & 0 deletions splunk_eventgen/lib/eventgensamples.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ class Sample(object):
# Internal fields
sampleLines = None
sampleDict = None
splunkEmbedded = False
_lockedSettings = None
_priority = None
_origName = None
Expand Down
35 changes: 21 additions & 14 deletions splunk_eventgen/lib/plugins/generator/jinja.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def __init__(self, msg):
self.msg = msg
super(CantFindTemplate, self).__init__(msg)


class CantProcessTemplate(Exception):
def __init__(self, msg):
"""Exception raised when we / Jinja can't find the template
Expand Down Expand Up @@ -188,30 +189,35 @@ def gen(self, count, earliest, latest, samplename=None):
startTime = datetime.datetime.now()

# if eventgen is running as Splunk app the configfile is None
if self.config.configfile:
working_dir, working_config_file = os.path.split(self.config.configfile)
else:
sample_dir = self._sample.sampleDir
if self._sample.splunkEmbedded is True:
splunk_home = os.environ["SPLUNK_HOME"]
app_name = getattr(self._sample, 'app', 'SA-Eventgen')
working_dir = os.path.join(splunk_home, 'etc', 'apps', app_name, 'default')
sample_dir = os.path.join(splunk_home, 'etc', 'apps', app_name,
'default', self._sample.DEFAULT_SAMPLE_DIR)

if not hasattr(self._sample, "jinja_template_dir"):
template_dir = os.path.join(os.path.dirname(working_dir), 'samples', 'templates')
template_dir = 'templates'
else:
template_dir = self._sample.jinja_template_dir

if not os.path.isabs(template_dir):
target_template_dir = os.path.join(working_dir, template_dir)
target_template_dir = os.path.join(sample_dir, template_dir)
else:
target_template_dir = template_dir
self.logger.info('set jinja template path to %s', target_template_dir)

if not hasattr(self._sample, "jinja_target_template"):
raise CantFindTemplate("Template to load not specified in eventgen conf for stanza. Skipping Stanza")
jinja_env = Environment(
loader=FileSystemLoader([target_template_dir, working_dir, template_dir], encoding='utf-8',
followlinks=False), extensions=[
'jinja2.ext.do', 'jinja2.ext.with_', 'jinja2.ext.loopcontrols', JinjaTime],
line_statement_prefix="#", line_comment_prefix="##")
loader=FileSystemLoader(
[target_template_dir], encoding='utf-8', followlinks=False),
extensions=[
'jinja2.ext.do', 'jinja2.ext.with_', 'jinja2.ext.loopcontrols',
JinjaTime
],
line_statement_prefix="#",
line_comment_prefix="##")

jinja_loaded_template = jinja_env.get_template(str(self._sample.jinja_target_template))
if hasattr(self._sample, 'jinja_variables'):
Expand All @@ -238,8 +244,11 @@ def gen(self, count, earliest, latest, samplename=None):
self.jinja_stream = jinja_loaded_template.stream(jinja_loaded_vars)
lines_out = []
try:
for line in self.jinja_stream:
if line != "\n":
for raw_line in self.jinja_stream:
# trim the newline char for jinja output
# it is quite normal to output empty newlines in jinja
line = raw_line.strip()
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need to trim the newline and empty lines for the output of jinja template.
Because for each validate event, we assume it is a one line json string which has the format

{ 'time': xxx, 'raw': xxx, 'sourcetype': sss}

skipping empty lines and breaking the loop is the root cause of #174

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how do I do multiline events then? We don't want to trim "every" "\n" just the ones where the only thing in the line is "\n".

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see the issue and get what you're trying to solve, but I don't think we can just use .strip(), gotta be another way

Copy link
Contributor Author

@GordonWang GordonWang Apr 24, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To generate multiline events in the ‘raw’ field in one json formatted events, use can do this in the jinja template directly. Just like this

{% for _ in range(0, large_number) %}
{%- time_now -%}
 {"_time":"{{ time_now_epoch }}", "_raw":"{{ time_now_formatted }}this is the first line, seq: {{loop.index}}/{{large_number}}\nthis is the second line, seq: {{loop.index}}/{{large_number}}\nthis is the thrid line, seq: {{loop.index}}/{{large_number}}"}
{% endfor %}

the output of this jinja template is

2019-04-24T10:47:35this is the first line, seq: 1/4
this is the second line, seq: 1/4
this is the thrid line, seq: 1/4
2019-04-24T10:47:35this is the first line, seq: 2/4
this is the second line, seq: 2/4
this is the thrid line, seq: 2/4
2019-04-24T10:47:35this is the first line, seq: 3/4
this is the second line, seq: 3/4
this is the thrid line, seq: 3/4
2019-04-24T10:47:35this is the first line, seq: 4/4
this is the second line, seq: 4/4
this is the thrid line, seq: 4/4

Make sense?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I should also say, are we sure we want to force people to write this single line, instead of letting them just hit return on the template? https://github.com/splunk/eventgen/blob/develop/tests/sample_eventgen_conf/jinja/templates/test_jinja_loop.template#L4. Specifically, doesn't it make sense to just hit return and put them on a new line?

Copy link
Contributor Author

@GordonWang GordonWang Apr 24, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No worries. User can hit the enter key to make the new line directly in jinja template file.
Jinja engine will keep what it is in your template file and render the output. No changes for newline and whitespace.

So, this template

{% for _ in range(0, large_number) %}
{%- time_now -%}
  {"_time":"{{ time_now_epoch }}", "_raw":"{{ time_now_formatted }}this is the first line, seq: {{loop.index}}/{{large_number}} 
    this is the second line, seq: {{loop.index}}/{{large_number}} 
    this is the thrid line, seq: {{loop.index}}/{{large_number}}"}
{% endfor %}

the output is

2019-04-24T15:58:23this is the first line, seq: 1/4
    this is the second line, seq: 1/4
    this is the thrid line, seq: 1/4
2019-04-24T15:58:23this is the first line, seq: 2/4
    this is the second line, seq: 2/4
    this is the thrid line, seq: 2/4
2019-04-24T15:58:23this is the first line, seq: 3/4
    this is the second line, seq: 3/4
    this is the thrid line, seq: 3/4
2019-04-24T15:58:23this is the first line, seq: 4/4
    this is the second line, seq: 4/4
    this is the thrid line, seq: 4/4

if line:
# TODO: Time can be supported by self._sample.timestamp, should probably set that up here.
try:
target_line = json.loads(line)
Expand Down Expand Up @@ -268,8 +277,6 @@ def gen(self, count, earliest, latest, samplename=None):
if "index" not in current_line_keys:
target_line["index"] = self._sample.index
lines_out.append(target_line)
else:
break
except TypeError as e:
self.logger.exception(e)
self.end_of_cycle = True
Expand Down
1 change: 0 additions & 1 deletion splunk_eventgen/lib/plugins/rater/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import logging
import logging.handlers
import random
import os


class ConfigRater(object):
Expand Down
Loading