diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..5da64a0 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/copyright/profiles_settings.xml b/.idea/copyright/profiles_settings.xml new file mode 100644 index 0000000..e7bedf3 --- /dev/null +++ b/.idea/copyright/profiles_settings.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml b/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml new file mode 100644 index 0000000..30ff5cb --- /dev/null +++ b/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__asm_asm_3_1.xml b/.idea/libraries/Maven__asm_asm_3_1.xml new file mode 100644 index 0000000..3386f10 --- /dev/null +++ b/.idea/libraries/Maven__asm_asm_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml b/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml new file mode 100644 index 0000000..0e66824 --- /dev/null +++ b/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml b/.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml new file mode 100644 index 0000000..01a573a --- /dev/null +++ b/.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_inject_extensions_guice_servlet_3_0.xml b/.idea/libraries/Maven__com_google_inject_extensions_guice_servlet_3_0.xml new file mode 100644 index 0000000..f8024fe --- /dev/null +++ b/.idea/libraries/Maven__com_google_inject_extensions_guice_servlet_3_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_inject_guice_3_0.xml b/.idea/libraries/Maven__com_google_inject_guice_3_0.xml new file mode 100644 index 0000000..b5a7a25 --- /dev/null +++ b/.idea/libraries/Maven__com_google_inject_guice_3_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml b/.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml new file mode 100644 index 0000000..c06d3a1 --- /dev/null +++ b/.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml b/.idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml new file mode 100644 index 0000000..a2a0c04 --- /dev/null +++ b/.idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml b/.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml new file mode 100644 index 0000000..880be15 --- /dev/null +++ b/.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_contribs_jersey_guice_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_contribs_jersey_guice_1_9.xml new file mode 100644 index 0000000..ff27f2e --- /dev/null +++ b/.idea/libraries/Maven__com_sun_jersey_contribs_jersey_guice_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_client_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_client_1_9.xml new file mode 100644 index 0000000..4c22f08 --- /dev/null +++ b/.idea/libraries/Maven__com_sun_jersey_jersey_client_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml new file mode 100644 index 0000000..abd3e5d --- /dev/null +++ b/.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml new file mode 100644 index 0000000..fff6aea --- /dev/null +++ b/.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml new file mode 100644 index 0000000..717467f --- /dev/null +++ b/.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_mail_javax_mail_1_5_2.xml b/.idea/libraries/Maven__com_sun_mail_javax_mail_1_5_2.xml new file mode 100644 index 0000000..11ef9e9 --- /dev/null +++ b/.idea/libraries/Maven__com_sun_mail_javax_mail_1_5_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml b/.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml new file mode 100644 index 0000000..1165a3b --- /dev/null +++ b/.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml b/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml new file mode 100644 index 0000000..3807eb2 --- /dev/null +++ b/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml new file mode 100644 index 0000000..1cb64ed --- /dev/null +++ b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml new file mode 100644 index 0000000..3fda2c0 --- /dev/null +++ b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml b/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml new file mode 100644 index 0000000..cec2493 --- /dev/null +++ b/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_codec_commons_codec_1_4.xml b/.idea/libraries/Maven__commons_codec_commons_codec_1_4.xml new file mode 100644 index 0000000..2b149c4 --- /dev/null +++ b/.idea/libraries/Maven__commons_codec_commons_codec_1_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml b/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml new file mode 100644 index 0000000..3caee7e --- /dev/null +++ b/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml b/.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml new file mode 100644 index 0000000..ae96781 --- /dev/null +++ b/.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml b/.idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml new file mode 100644 index 0000000..d44fc67 --- /dev/null +++ b/.idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml b/.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml new file mode 100644 index 0000000..49f1def --- /dev/null +++ b/.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_el_commons_el_1_0.xml b/.idea/libraries/Maven__commons_el_commons_el_1_0.xml new file mode 100644 index 0000000..598d1b0 --- /dev/null +++ b/.idea/libraries/Maven__commons_el_commons_el_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml b/.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml new file mode 100644 index 0000000..66e6537 --- /dev/null +++ b/.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_io_commons_io_2_4.xml b/.idea/libraries/Maven__commons_io_commons_io_2_4.xml new file mode 100644 index 0000000..bc2aad0 --- /dev/null +++ b/.idea/libraries/Maven__commons_io_commons_io_2_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml b/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml new file mode 100644 index 0000000..2ec8376 --- /dev/null +++ b/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml b/.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml new file mode 100644 index 0000000..01c7b8e --- /dev/null +++ b/.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_net_commons_net_3_1.xml b/.idea/libraries/Maven__commons_net_commons_net_3_1.xml new file mode 100644 index 0000000..dc40661 --- /dev/null +++ b/.idea/libraries/Maven__commons_net_commons_net_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__io_netty_netty_3_6_2_Final.xml b/.idea/libraries/Maven__io_netty_netty_3_6_2_Final.xml new file mode 100644 index 0000000..f72cc59 --- /dev/null +++ b/.idea/libraries/Maven__io_netty_netty_3_6_2_Final.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_activation_activation_1_1_1.xml b/.idea/libraries/Maven__javax_activation_activation_1_1_1.xml new file mode 100644 index 0000000..bde4d35 --- /dev/null +++ b/.idea/libraries/Maven__javax_activation_activation_1_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_inject_javax_inject_1.xml b/.idea/libraries/Maven__javax_inject_javax_inject_1.xml new file mode 100644 index 0000000..93cf65a --- /dev/null +++ b/.idea/libraries/Maven__javax_inject_javax_inject_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml b/.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml new file mode 100644 index 0000000..4b5f67f --- /dev/null +++ b/.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml b/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml new file mode 100644 index 0000000..679e09a --- /dev/null +++ b/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml b/.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml new file mode 100644 index 0000000..a3054d2 --- /dev/null +++ b/.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml b/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml new file mode 100644 index 0000000..2a4dd7a --- /dev/null +++ b/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__log4j_log4j_1_2_17.xml b/.idea/libraries/Maven__log4j_log4j_1_2_17.xml new file mode 100644 index 0000000..e383c1b --- /dev/null +++ b/.idea/libraries/Maven__log4j_log4j_1_2_17.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml b/.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml new file mode 100644 index 0000000..f4ff312 --- /dev/null +++ b/.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml b/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml new file mode 100644 index 0000000..cc03056 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml new file mode 100644 index 0000000..190209e --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_email_1_4.xml b/.idea/libraries/Maven__org_apache_commons_commons_email_1_4.xml new file mode 100644 index 0000000..97d41c2 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_email_1_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml new file mode 100644 index 0000000..5627f0c --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_4_0.xml new file mode 100644 index 0000000..4b2be4a --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_4_0.xml new file mode 100644 index 0000000..e3bbfea --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_4_0.xml new file mode 100644 index 0000000..51248c6 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_4_0.xml new file mode 100644 index 0000000..2d0fdba --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_4_0.xml new file mode 100644 index 0000000..da3606b --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_4_0.xml new file mode 100644 index 0000000..9315acc --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_4_0.xml new file mode 100644 index 0000000..910e63a --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_4_0.xml new file mode 100644 index 0000000..a4ff7cf --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_4_0.xml new file mode 100644 index 0000000..557bb54 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_4_0.xml new file mode 100644 index 0000000..0ae046a --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_4_0.xml new file mode 100644 index 0000000..21d2f0d --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_4_0.xml new file mode 100644 index 0000000..dc09aff --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_nodemanager_2_4_0.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_nodemanager_2_4_0.xml new file mode 100644 index 0000000..4298a61 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_nodemanager_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_1_2.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_1_2.xml new file mode 100644 index 0000000..1e5c3b5 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_1_2.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_1_2.xml new file mode 100644 index 0000000..9548c49 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_5.xml b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_5.xml new file mode 100644 index 0000000..3019b0a --- /dev/null +++ b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_8_8.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_8_8.xml new file mode 100644 index 0000000..d649056 --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_8_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_8_3.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_8_3.xml new file mode 100644 index 0000000..f3e0cf6 --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_8_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_8_8.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_8_8.xml new file mode 100644 index 0000000..148cdfd --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_8_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml new file mode 100644 index 0000000..9fc3894 --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml b/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml new file mode 100644 index 0000000..d62802e --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_json_json_20090211.xml b/.idea/libraries/Maven__org_json_json_20090211.xml new file mode 100644 index 0000000..e57dc0a --- /dev/null +++ b/.idea/libraries/Maven__org_json_json_20090211.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml new file mode 100644 index 0000000..0f5d0b8 --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml new file mode 100644 index 0000000..f6ec2b9 --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_5.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_5.xml new file mode 100644 index 0000000..6f808f1 --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_5.xml b/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_5.xml new file mode 100644 index 0000000..df1d383 --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_tukaani_xz_1_0.xml b/.idea/libraries/Maven__org_tukaani_xz_1_0.xml new file mode 100644 index 0000000..8b8042f --- /dev/null +++ b/.idea/libraries/Maven__org_tukaani_xz_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_0_4_1.xml b/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_0_4_1.xml new file mode 100644 index 0000000..ee11787 --- /dev/null +++ b/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_0_4_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml b/.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml new file mode 100644 index 0000000..4610fa2 --- /dev/null +++ b/.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml b/.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml new file mode 100644 index 0000000..e80db89 --- /dev/null +++ b/.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml b/.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml new file mode 100644 index 0000000..59d176d --- /dev/null +++ b/.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..233a7f0 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..82a22b1 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/hma-v2.0.iml b/hma-v2.0.iml new file mode 100644 index 0000000..dfc1350 --- /dev/null +++ b/hma-v2.0.iml @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..9db3d4a --- /dev/null +++ b/pom.xml @@ -0,0 +1,46 @@ + + + 4.0.0 + + org.hhxy.hma2.0 + hma + 1.0-SNAPSHOT + + + org.apache.hadoop + hadoop-common + 2.4.0 + + + org.apache.hadoop + hadoop-hdfs + 2.4.0 + + + org.apache.hadoop + hadoop-mapreduce-client-core + 2.4.0 + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + 2.4.0 + + + org.json + json + 20090211 + + + org.apache.commons + commons-email + 1.4 + + + + + + + \ No newline at end of file diff --git a/src/META-INF/MANIFEST.MF b/src/META-INF/MANIFEST.MF new file mode 100644 index 0000000..72bb728 --- /dev/null +++ b/src/META-INF/MANIFEST.MF @@ -0,0 +1,34 @@ +Manifest-Version: 1.0 +Class-Path: wsdl4j-1.6.2.jar zookeeper-3.4.6.jar netty-all-4.0.8.Final + .jar guice-3.0.jar servlet-api-2.5.jar jettison-1.1.jar jaxws-api-2.2 + .6.jar jsch-0.1.42.jar saaj-api-1.3.4.jar gson-2.2.2.jar javax.annota + tion-3.1.1.jar hadoop-yarn-client-2.4.0.jar jets3t-0.9.0.jar json-lib + -2.4-jdk15.jar jaxb-impl-2.2.3-1.jar htrace-core-3.1.0-incubating.jar + jetty-6.1.26.jar guice-servlet-3.0.jar hadoop-common-2.4.0.jar jerse + y-json-1.9.jar commons-compress-1.4.1.jar ant-1.6.5.jar commons-codec + -1.4.jar axis-1.4.jar hbase-common-1.0.0.jar xz-1.0.jar commons-httpc + lient-3.1.jar jersey-core-1.9.jar log4j-1.2.16.jar commons-configurat + ion-1.6.jar hbase-hadoop-compat-1.0.0.jar commons-collections-3.2.1.j + ar hadoop-auth-2.4.0.jar ezmorph-1.0.6.jar javax.xml.rpc-api-1.1.jar + guava-14.0.jar commons-beanutils-1.8.0.jar aopalliance-1.0.jar mysql- + connector-java-5.1.15.jar commons-discovery-0.5.jar slf4j-api-1.6.6.j + ar jackson-core-asl-1.0.1.jar commons-lang-2.4.jar xmlenc-0.52.jar st + ax-api-1.0-2.jar paranamer-2.3.jar antlr-runtime-3.0.1.jar jersey-gui + ce-1.9.jar jasper-compiler-5.5.23.jar proxool-cglib-1.0.jar jackson-m + apper-asl-1.0.1.jar stringtemplate-3.1-b1.jar hadoop-yarn-server-comm + on-2.4.0.jar jersey-client-1.9.jar javax.inject-1.jar commons-cli-1.2 + .jar jsr181-api-1.0-MR1.jar hadoop-mapreduce-client-core-2.4.0.jar hb + ase-server-1.0.0.jar hadoop-annotations-2.4.0.jar commons-el-1.0.jar + hadoop-yarn-api-2.4.0.jar activation-1.1.jar jasper-runtime-5.5.23.ja + r snappy-java-1.0.4.1.jar hadoop-mapreduce-client-common-2.4.0.jar ja + vax.jws-10.0-b28.jar netty-3.2.4.Final.jar commons-net-3.1.jar netty- + 3.6.2.Final.jar jsr305-1.3.9.jar httpclient-4.1.2.jar java-xmlbuilder + -0.4.jar avro-1.7.4.jar commons-beanutils-core-1.8.0.jar commons-dige + ster-1.8.jar slf4j-log4j12-1.6.6.jar jsp-api-2.0.jar hbase-client-1.0 + .0.jar jetty-util-6.1.26.jar jsp-api-2.1.jar jersey-server-1.9.jar ja + ckson-jaxrs-1.8.3.jar hadoop-yarn-common-2.4.0.jar asm-3.1.jar common + s-math3-3.1.1.jar httpcore-4.1.2.jar jaxb-api-2.2.4.jar jackson-xc-1. + 8.3.jar proxool-0.9.1.jar commons-logging-1.1.1.jar hbase-protocol-1. + 0.0.jar protobuf-java-2.5.0.jar commons-io-2.4.jar +Main-Class: com.sina.data.bigmonitor.web.HttpServer + diff --git a/src/compile.sh b/src/compile.sh new file mode 100644 index 0000000..8e7b574 --- /dev/null +++ b/src/compile.sh @@ -0,0 +1,29 @@ +#!/bin/bash - +pwd=$(cd $(dirname $0);pwd) +HMAPATH="${pwd}/../../" +CLASSPATH="${HMAPATH}/hma/src" + +# add libs to CLASSPATH +for f in ${HMAPATH}/hma/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +for f in ${HMAPATH}/hma/lib/*/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +for f in ${HMAPATH}/hma/lib/*/*/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +${HMAPATH}/java8/bin/javac */*.java -cp ${CLASSPATH} +${HMAPATH}/java8/bin/javac */*/*.java -cp ${CLASSPATH} +${HMAPATH}/java8/bin/javac */*/*/*.java -cp ${CLASSPATH} +${HMAPATH}/java8/bin/javac */*/*/*/*.java -cp ${CLASSPATH} +${HMAPATH}/java8/bin/javac */*/*/*/*/*.java -cp ${CLASSPATH} + +${HMAPATH}/java8/bin/jar cf hma.jar hma/ +#${HMAPATH}/java8/bin/jar cf json.jar org/ + +#cp hma.jar json.jar ../lib.http/ +mv hma.jar ../lib/ diff --git a/src/java/hma.zip b/src/java/hma.zip new file mode 100644 index 0000000..cfee182 Binary files /dev/null and b/src/java/hma.zip differ diff --git a/src/java/hma/conf/Configurable.java b/src/java/hma/conf/Configurable.java new file mode 100644 index 0000000..a525819 --- /dev/null +++ b/src/java/hma/conf/Configurable.java @@ -0,0 +1,13 @@ +package hma.conf; + +import hma.conf.Configuration; + +/** Something that may be configured with a {@link Configuration}. */ +public interface Configurable { + + /** Set the configuration to be used by this object. */ + void setConf(Configuration conf); + + /** Return the configuration used by this object. */ + Configuration getConf(); +} diff --git a/src/java/hma/conf/Configuration.java b/src/java/hma/conf/Configuration.java new file mode 100644 index 0000000..8c6d6a6 --- /dev/null +++ b/src/java/hma/conf/Configuration.java @@ -0,0 +1,626 @@ +package hma.conf; + +import hma.util.LOG; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; +import org.xml.sax.SAXException; + + +/** + * @author Vergil + * + */ +public class Configuration implements Iterable> { + + /** + * List of configuration resources. + */ + private List resources = new ArrayList(); + + private Properties properties; + + //private boolean quietmode = true; + private boolean quietmode = false; + + private String propertyNameTagString = "name"; + private String propertyValueTagString = "value"; + + + private ClassLoader classLoader; + { + classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = Configuration.class.getClassLoader(); + } + } + + + private String moduleName = null; + private boolean loadDefaults = true; + + /** A new configuration. */ + public Configuration() { + this.initConfigurationResources(); + } + + /** + * A new configuration where the behavior of reading from the default + * resources can be turned off. + * + * If the parameter {@code loadDefaults} is false, the new instance + * will not load resources from the default files. + * @param loadDefaults specifies whether to load from the default files + */ + public Configuration(boolean loadDefaults) { + this.loadDefaults = loadDefaults; + this.initConfigurationResources(); + } + + public Configuration(boolean loadDefaults, String moduleName) { + this.loadDefaults = loadDefaults; + this.moduleName = moduleName; + this.initConfigurationResources(); + } + + private synchronized void initConfigurationResources() { + if ((this.moduleName != null) && this.loadDefaults) { + resources.add(moduleName + "-default.xml"); + //resources.add(moduleName + "-site.xml"); + //resources.add("D:\\eclipse\\guoyezhi\\workspace\\HMA\\conf\\hma-default.xml"); + } + } + + public synchronized void reinitConfigurationResources() { + resources.clear(); + if ((this.moduleName != null) && this.loadDefaults) { + resources.add(moduleName + "-default.xml"); + resources.add(moduleName + "-site.xml"); + //resources.add("D:\\eclipse\\guoyezhi\\workspace\\HMA\\src\\monitor-items-site.xml"); + } + this.reloadConfiguration(); + } + + private synchronized void addConfigurationResourceObject(Object resource) { + resources.add(resource); // add to resources + reloadConfiguration(); + } + + /** + * Add a configuration resource. + * + * The properties of this resource will override properties of previously + * added resources, unless they were marked final. + * + * @param name + * resource to be added, the classpath is examined for a file + * with that name. + */ + public void addConfigurationResource(String name) { + addConfigurationResourceObject(name); + } + + public void addConfigurationResource(File file) { + addConfigurationResourceObject(file); + } + + /** + * Reload configuration from previously added resources. + * + * This method will clear all the configuration read from the added + * resources, and final parameters. This will make the resources to + * be read again before accessing the values. Values that are added + * via set methods will overlay values read from the resources. + */ + public synchronized void reloadConfiguration() { + properties = null; // trigger reload + } + + public synchronized void setModuleName(String moduleName) { + if (moduleName != null) + this.moduleName = moduleName; + this.reinitConfigurationResources(); + } + + + protected synchronized List getResources() { + return resources; + } + + protected synchronized void setResources(List resources) { + this.resources = resources; + } + + public synchronized boolean getQuietMode() { + return quietmode; + } + + /** + * Set the quiteness-mode. + * + * In the the quite-mode error and informational messages might not be logged. + * + * @param quietmode true to set quiet-mode on, false + * to turn it off. + */ + public synchronized void setQuietMode(boolean quietmode) { + this.quietmode = quietmode; + } + + public synchronized String getPropertyNameTagString() { + return propertyNameTagString; + } + + public synchronized void setPropertyNameTagString(String nameTagStr) { + this.propertyNameTagString = nameTagStr; + } + + public synchronized String getPropertyValueTagString() { + return propertyValueTagString; + } + + public synchronized void setPropertyValueTagString(String valueTagStr) { + this.propertyValueTagString = valueTagStr; + } + + + private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); + private static int MAX_SUBST = 40; + + private String substituteVars(String expr) { + if (expr == null) { + return null; + } + Matcher match = varPat.matcher(""); + String eval = expr; + for(int s=0; sname property, null if + * no such property exists. + * + * Values are processed for variable expansion + * before being returned. + * + * @param name the property name. + * @return the value of the name property, + * or null if no such property exists. + */ + public String get(String name) { + return substituteVars(getProps().getProperty(name)); + } + + /** + * Get the value of the name property. If no such property + * exists, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value, or defaultValue if the property + * doesn't exist. + */ + public String get(String name, String defaultValue) { + String res = substituteVars(getProps().getProperty(name, defaultValue)); + if (res == null) { + System.err.println("Vergil DEBUG: ${" + + name + "} not found!"); + } + return res; + } + + /** + * Get the comma delimited values of the name property as an + * array of Strings. If no such property is specified then + * null is returned. + * + * @param name + * property name. + * @return property value as an array of Strings, or + * null. + */ + public String[] getStrings(String name) { + String valueString = get(name); + if (valueString == null) + return null; + return valueString.split(","); + } + + /** + * Get the value of the name property, without doing + * variable expansion. + * + * @param name the property name. + * @return the value of the name property, + * or null if no such property exists. + */ + public String getRaw(String name) { + return getProps().getProperty(name); + } + + private String getHexDigits(String value) { + boolean negative = false; + String str = value; + String hexString = null; + if (value.startsWith("-")) { + negative = true; + str = value.substring(1); + } + if (str.startsWith("0x") || str.startsWith("0X")) { + hexString = str.substring(2); + if (negative) { + hexString = "-" + hexString; + } + return hexString; + } + return null; + } + + /** + * Get the value of the name property as an int. + * + * If no such property exists, or if the specified value is not a valid + * int, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value as an int, + * or defaultValue. + */ + public int getInt(String name, int defaultValue) { + String valueString = get(name); + if (valueString == null) + return defaultValue; + try { + String hexString = getHexDigits(valueString); + if (hexString != null) { + return Integer.parseInt(hexString, 16); + } + return Integer.parseInt(valueString); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + /** + * Get the value of the name property as a long. + * If no such property is specified, or if the specified value is not a valid + * long, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value as a long, + * or defaultValue. + */ + public long getLong(String name, long defaultValue) { + String valueString = get(name); + if (valueString == null) + return defaultValue; + try { + String hexString = getHexDigits(valueString); + if (hexString != null) { + return Long.parseLong(hexString, 16); + } + return Long.parseLong(valueString); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + /** + * Get the value of the name property as a float. + * If no such property is specified, or if the specified value is not a valid + * float, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value as a float, + * or defaultValue. + */ + public float getFloat(String name, float defaultValue) { + String valueString = get(name); + if (valueString == null) + return defaultValue; + try { + return Float.parseFloat(valueString); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + /** + * Get the value of the name property as a boolean. + * If no such property is specified, or if the specified value is not a valid + * boolean, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value as a boolean, + * or defaultValue. + */ + public boolean getBoolean(String name, boolean defaultValue) { + String valueString = get(name); + if ("true".equals(valueString)) + return true; + else if ("false".equals(valueString)) + return false; + else + return defaultValue; + } + + /** + * Load a class by name. + * + * @param name the class name. + * @return the class object. + * @throws ClassNotFoundException if the class is not found. + */ + public Class getClassByName(String name) throws ClassNotFoundException { + return Class.forName(name, true, classLoader); + } + + /** + * Get the value of the name property as a Class. + * If no such property is specified, then defaultValue is + * returned. + * + * @param name the class name. + * @param defaultValue default value. + * @return property value as a Class, + * or defaultValue. + */ + public Class getClass(String name, Class defaultValue) { + String valueString = get(name); + if (valueString == null) + return defaultValue; + try { + return getClassByName(valueString); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + + private synchronized Properties getProps() { + + List list = null; + + if (properties == null) { + properties = new Properties(); + list = loadResources(resources, quietmode); + + Iterator iter = list.iterator(); + while(iter.hasNext()) { + Element prop = iter.next(); + if (!"property".equals(prop.getTagName())) { + //if (!"monitor_item".equals(prop.getTagName())) + // LOG.warn("bad conf file: element not "); + continue; + } + + NodeList fields = prop.getChildNodes(); + String name = null; + String value = null; + for (int i = 0; i < fields.getLength(); i++) { + Node fieldNode = fields.item(i); + if (!(fieldNode instanceof Element)) + continue; + Element field = (Element)fieldNode; + if (propertyNameTagString.equals(field.getTagName()) && field.hasChildNodes()) + name = ((Text)field.getFirstChild()).getData().trim(); + if (propertyValueTagString.equals(field.getTagName()) && field.hasChildNodes()) + value = ((Text)field.getFirstChild()).getData(); + } + + if (name != null && value != null) { + properties.setProperty(name, value); + } + } + } + + return properties; + } + + protected List loadResources(List resources, boolean quiet) { + LinkedList list = new LinkedList(); + for (Object resource : resources) { + list.addAll(loadResource(resource, quiet)); + } + return list; + } + + /** + * Get the {@link URL} for the named resource. + * + * @param name + * resource name. + * @return the url for the named resource. + */ + public URL locateResource(String name) { + return classLoader.getResource(name); + } + + protected List loadResource(Object name, boolean quiet) { + + LinkedList list = new LinkedList(); + + try { + DocumentBuilderFactory docBuilderFactory = + DocumentBuilderFactory.newInstance(); + //ignore all comments inside the xml file + docBuilderFactory.setIgnoringComments(true); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = null; + + if (name instanceof URL) { // an URL resource + URL url = (URL)name; + if (url != null) { + if (!quiet) { + LOG.info("parsing " + url); + } + doc = builder.parse(url.toString()); + } + } else if (name instanceof String) { // a file resource + + /* + File file = new File((String)name); + if (file.exists()) { + if (!quiet) { + LOG.info("parsing " + file); + } + InputStream in = new BufferedInputStream(new FileInputStream(file)); + try { + doc = builder.parse(in); + } finally { + in.close(); + } + } + */ + + /* + * TODO: used in released build + */ + URL url = locateResource((String) name); + if (url != null) { + if (!quiet) { + LOG.info("parsing " + url); + } + doc = builder.parse(url.toString()); + } + + } else if (name instanceof File) { + + File file = (File) name; + if (file.exists()) { + if (!quiet) { + LOG.info("parsing " + file); + } + InputStream in = new BufferedInputStream(new FileInputStream(file)); + try { + doc = builder.parse(in); + } finally { + in.close(); + } + } + + } else if (name instanceof InputStream) { + try { + doc = builder.parse((InputStream)name); + } finally { + ((InputStream)name).close(); + } + } else { + LOG.debug("here"); + } + + if (doc == null) { + if (quiet) + return null; + throw new RuntimeException(name + " not found"); + } + + Element root = doc.getDocumentElement(); + if (!"configuration".equals(root.getTagName())) + LOG.fatal("bad conf file: top-level element not "); + NodeList props = root.getChildNodes(); + for (int i = 0; i < props.getLength(); i++) { + Node propNode = props.item(i); + if (!(propNode instanceof Element)) + continue; + Element prop = (Element)propNode; + list.add(prop); + } + + } catch (IOException e) { + LOG.fatal("error parsing conf file: " + e); + throw new RuntimeException(e); + } catch (SAXException e) { + LOG.fatal("error parsing conf file: " + e); + throw new RuntimeException(e); + } catch (ParserConfigurationException e) { + LOG.fatal("error parsing conf file: " + e); + throw new RuntimeException(e); + } + + return list; + } + + + /** + * Get an {@link Iterator} to go through the list of String + * key-value pairs in the configuration. + * + * @return an iterator over the entries. + */ + @Override + public Iterator> iterator() { + Map result = new HashMap(); + for (Map.Entry item : getProps().entrySet()) { + if (item.getKey() instanceof String + && item.getValue() instanceof String) { + result.put((String) item.getKey(), (String) item.getValue()); + } + } + return result.entrySet().iterator(); + } + + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + System.out.print(System.getenv()); + Configuration conf = new Configuration(true); + conf.setPropertyValueTagString("classpath"); + conf.getProps().list(System.out); + } + +} diff --git a/src/java/hma/conf/Configured.java b/src/java/hma/conf/Configured.java new file mode 100644 index 0000000..aa99791 --- /dev/null +++ b/src/java/hma/conf/Configured.java @@ -0,0 +1,28 @@ +package hma.conf; + +/** Base class for things that may be configured with a {@link Configuration}. */ +public class Configured implements Configurable { + + private Configuration conf; + + /** Construct a Configured. */ + public Configured() { + this(null); + } + + /** Construct a Configured. */ + public Configured(Configuration conf) { + setConf(conf); + } + + // inherit javadoc + public void setConf(Configuration conf) { + this.conf = conf; + } + + // inherit javadoc + public Configuration getConf() { + return conf; + } + +} diff --git a/src/java/hma/monitor/HMAMonitorProtocol.java b/src/java/hma/monitor/HMAMonitorProtocol.java new file mode 100644 index 0000000..b5849fd --- /dev/null +++ b/src/java/hma/monitor/HMAMonitorProtocol.java @@ -0,0 +1,18 @@ +package hma.monitor; + +import org.apache.hadoop.ipc.VersionedProtocol; +/** + * Created with IntelliJ IDEA. + * User: uuleon + * Date: 13-4-28 + * Time: 下午12:53 + * To change this template use File | Settings | File Templates. + */ +public interface HMAMonitorProtocol extends VersionedProtocol { + + public boolean suspendAlarm(String cluster); + + public boolean recoverAlarm(String cluster); + + public MonitorStatus getAlarmStatus(String cluster); +} diff --git a/src/java/hma/monitor/MonitorItem.java b/src/java/hma/monitor/MonitorItem.java new file mode 100644 index 0000000..1ae379e --- /dev/null +++ b/src/java/hma/monitor/MonitorItem.java @@ -0,0 +1,42 @@ +/** + * + */ +package hma.monitor; + +import hma.monitor.collection.MonitorItemWorker; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemPrototype; + +/** + * @author guoyezhi + * + */ +public class MonitorItem { + + private MonitorItemWorker worker = null; + + private MonitorItemPrototype prototype = null; + + public MonitorItem(MonitorItemWorker worker, + MonitorItemPrototype prototype) { + this.worker = worker; + this.prototype = prototype; + } + + public void setPrototype(MonitorItemPrototype prototype) { + this.prototype = prototype; + } + + public MonitorItemPrototype getPrototype() { + return prototype; + } + + public MonitorItemData collect() { + return worker.collect(); + } + + public MonitorItemData extract(MonitorItemData rawData) { + return worker.extract(rawData); + } + +} diff --git a/src/java/hma/monitor/MonitorItemCollector.java b/src/java/hma/monitor/MonitorItemCollector.java new file mode 100644 index 0000000..c02f4c4 --- /dev/null +++ b/src/java/hma/monitor/MonitorItemCollector.java @@ -0,0 +1,74 @@ +/** + * + */ +package hma.monitor; + +import java.util.Map; + +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemPrototype; +import hma.util.TimedTask; + +/** + * @author guoyezhi + * + */ +public class MonitorItemCollector extends TimedTask implements Runnable { + + final private MonitorItem item; + + final private MonitorItemPrototype prototype; + + final private Map> rawDataPool; + + public MonitorItemCollector( + MonitorItem item, + Map> rawDataPool) { + this.item = item; + this.prototype = item.getPrototype(); + this.rawDataPool = rawDataPool; + } + + /* + * (non-Javadoc) + * + * @see hma.util.TimedTask#run() + */ + @SuppressWarnings("unchecked") + @Override + public void run() { + + String rawDataTypeName = prototype.getRawDataTypeName(); + + //System.out.println("MonitorItemCollector.run() starting ... \t" + // + new java.util.Date()); + + /* + * MonitorItem#collect() may be a time-consuming method involving + * I/O operations, so we should not put it into synchronized block! + */ + MonitorItemData newRawData = item.collect(); + if (newRawData == null) { + // TODO: add WARN logs + return; + } + + synchronized (rawDataPool) { + MonitorItemData oldRawData = + (MonitorItemData) rawDataPool.get(rawDataTypeName); + if (oldRawData == null || + (oldRawData.getTimestamp() < newRawData.getTimestamp())) { + rawDataPool.put(rawDataTypeName, newRawData); + } else { + // TODO: add WARN logs + System.out.println( + "OldRawData.Timestamp >= NewRawData.Timestamp"); + } + } + + //System.out.println("MonitorItemCollector.run() ending ... \t" + // + new java.util.Date()); + + } + +} diff --git a/src/java/hma/monitor/MonitorItemExtractor.java b/src/java/hma/monitor/MonitorItemExtractor.java new file mode 100644 index 0000000..81c7df5 --- /dev/null +++ b/src/java/hma/monitor/MonitorItemExtractor.java @@ -0,0 +1,100 @@ +/** + * + */ +package hma.monitor; + +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemPrototype; +import hma.util.TimedTask; + +import java.util.Date; +import java.util.Map; + +/** + * @author guoyezhi + * + */ +public class MonitorItemExtractor extends TimedTask implements Runnable { + + final private MonitorItem item; + + final private MonitorItemPrototype prototype; + + final private Map> rawDataPool; + + final private Map> extractedDataPool; + + public MonitorItemExtractor( + MonitorItem item, + Map> rawDataPool, + Map> extractedDataPool) { + this.item = item; + this.prototype = item.getPrototype(); + this.rawDataPool = rawDataPool; + this.extractedDataPool = extractedDataPool; + } + + /* + * (non-Javadoc) + * + * @see hma.util.TimedTask#run() + */ + //@SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") + @Override + public void run() { + + String monitorItemName = prototype.getName(); + String rawDataTypeName = prototype.getRawDataTypeName(); + + //System.out.println(monitorItemName + " ---> " + System.currentTimeMillis()); + + MonitorItemData rawData = null; + synchronized (rawDataPool) { + rawData = + (MonitorItemData) rawDataPool.get(rawDataTypeName); + } + if (rawData == null) { + // TODO: add WARN logs + /* + System.out.println("RawDataPool.get(" + + rawDataTypeName + + ") == null"); + */ + return; + } + + MonitorItemData newExtractedData = item.extract(rawData); + if (newExtractedData == null) { + // TODO: add WARN logs + System.out.println("MonitorItem#extract(rawData) == null"); + return; + } + newExtractedData.setName(monitorItemName); + + synchronized (extractedDataPool) { + MonitorItemData oldExtractedData = + (MonitorItemData) extractedDataPool.get(monitorItemName); + if (oldExtractedData != null) { + long oldExtractedTime = oldExtractedData.getTimestamp(); + long newExtractedTime = newExtractedData.getTimestamp(); + if (oldExtractedTime > newExtractedTime) { + // TODO: add WARN logs + System.out.println(monitorItemName + + " : OldExtractedData.Timestamp " + + ">= NewExtractedData.Timestamp"); + return; + } else if (oldExtractedTime == newExtractedTime) { + // nothing needed to do + //System.out.println(monitorItemName + " ---< " + System.currentTimeMillis()); + return; + } + } + extractedDataPool.put(monitorItemName, newExtractedData); + System.out.println("put " + monitorItemName + " # " + new Date(newExtractedData.getTimestamp())); + } + + //System.out.println(monitorItemName + " ---< " + System.currentTimeMillis()); + } + +} diff --git a/src/java/hma/monitor/MonitorItemManager.java b/src/java/hma/monitor/MonitorItemManager.java new file mode 100644 index 0000000..d98e651 --- /dev/null +++ b/src/java/hma/monitor/MonitorItemManager.java @@ -0,0 +1,373 @@ +/** + * + */ +package hma.monitor; + +import hma.conf.Configured; +import hma.monitor.collection.BuiltinMonitorItemPrototype; +import hma.monitor.collection.MonitorItemConfiguration; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemPrototype; +import hma.monitor.collection.MonitorItemWorker; +import hma.monitor.collection.metric.CollectionMetricDBDumper; +import hma.util.BuiltinPrototypesInjector; +import hma.util.TimedTaskManager; + +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Random; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +/** + * @author guoyezhi + * add additional by chiwen01 + * + */ +public class MonitorItemManager extends Configured implements Runnable { + + /** + * Raw Data Pool of Monitored Items + */ + private Map> rawDataPool = + new HashMap>(); + + /** + * Target Data Pool of Monitored Items extracted from Raw Data Pool + */ + private Map> extractedDataPool = + new HashMap>(); + + private Map allBuiltinPrototypes = null; + @SuppressWarnings("unused") + private Map uniqueBuiltinPrototypesByRawData = null; + + private Map> allBuiltinWorkers = null; + // TODO: private Map> allPluginWorkers = null; + + private Map> allMonitorItems = null; + + private Map> allCollectors = null; + private Map> allExtractors = null; + + private TimedTaskManager builtinCollectorTimedScheduler = null; + // TODO: private TimedTaskManager pluginCollectorTimedScheduler = null; + private TimedTaskManager extractorTimedScheduler = null; + + + final private MonitorManager monitorManager; + + private static int monitorItemWorkerNum = 0; + + private static int extractionPeriod = 10; + + + private Random rand = null; + + + public MonitorItemManager() { + super(new MonitorItemConfiguration(true)); + monitorManager = MonitorManager.getMonitorManager(); + monitorItemWorkerNum = + monitorManager.getConf().getInt( + "monitor.item.worker.num", 1024); + extractionPeriod = + monitorManager.getConf().getInt( + "monitor.item.data.extraction.period", 1); + builtinCollectorTimedScheduler = + new TimedTaskManager(monitorItemWorkerNum); + extractorTimedScheduler = + new TimedTaskManager(monitorItemWorkerNum); + rand = new Random(); + } + + + private synchronized void initBuiltinMonitorItemWorkers() { + + MonitorItemConfiguration miConf = + (MonitorItemConfiguration) this.getConf(); + + if (allBuiltinPrototypes == null) { + allBuiltinPrototypes = miConf.getBuiltinPrototypes(); + BuiltinPrototypesInjector.BuiltinPrototypesInjector(allBuiltinPrototypes); + + } + + if (allBuiltinWorkers == null) { + allBuiltinWorkers = new HashMap>(); + } + + Iterator> iter = + allBuiltinPrototypes.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String name = entry.getKey(); + //System.out.println("kain's log about BuiltinItem : " + name); + BuiltinMonitorItemPrototype prototype = entry.getValue(); + MonitorItemWorker worker = null; + try { + worker = (MonitorItemWorker) + prototype.getMonitorItemWorkerClass().newInstance(); + worker.setMonitorItemPrototype(prototype); + worker.MyItemName = name; + + } catch (InstantiationException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + allBuiltinWorkers.put(name, worker); + } + + } + + @SuppressWarnings("unused") + private synchronized void initPluginMonitorItemWorkers() { + // TODO + } + + @SuppressWarnings("unchecked") + private synchronized void initAllMonitorItems() { + if (this.allBuiltinWorkers == null) { + this.initBuiltinMonitorItemWorkers(); + } + // TODO: if (this.allPluginWorkers == null) { } + + if (this.allMonitorItems == null) { + this.allMonitorItems = new HashMap>(); + } + + Iterator> iter = + this.allBuiltinPrototypes.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String name = entry.getKey(); + MonitorItemPrototype prototype = entry.getValue(); + MonitorItemWorker worker = this.allBuiltinWorkers.get(name); + if (prototype == null || worker == null) { + throw new RuntimeException( + "Prototype & Worker is not paired with the other."); + } + MonitorItem item = + (MonitorItem) new MonitorItem(worker, prototype); + this.allMonitorItems.put(name, item); + } + } + + @SuppressWarnings("unchecked") + private synchronized void initAllCollectors() { + if (this.allMonitorItems == null) { + this.initAllMonitorItems(); + } + if (this.allCollectors == null) { + this.allCollectors = new HashMap>(); + } + Iterator>> iter = + this.allMonitorItems.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + String name = entry.getKey(); + MonitorItem item = entry.getValue(); + MonitorItemCollector collector = + new MonitorItemCollector(item, rawDataPool); + this.allCollectors.put(name, collector); + } + } + + @SuppressWarnings("unchecked") + private synchronized void initAllExtractors() { + if (this.allMonitorItems == null) { + this.initAllMonitorItems(); + } + if (this.allExtractors == null) { + this.allExtractors = new HashMap>(); + } + Iterator>> iter = + this.allMonitorItems.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + String name = entry.getKey(); + MonitorItem item = entry.getValue(); + MonitorItemExtractor extractor = + new MonitorItemExtractor(item, rawDataPool, getExtractedDataPool()); + this.allExtractors.put(name, extractor); + } + } + + private synchronized void launchBuiltinCollectors () { + + Map uniqueBuiltinPrototypes = + new HashMap(); + + Iterator> iter = + this.allBuiltinPrototypes.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + BuiltinMonitorItemPrototype prototype = entry.getValue(); + String rawDataTypeName = prototype.getRawDataTypeName(); + String name = prototype.getName(); + System.out.println("kain's log about builtin prototyp name : " + name ); +// BuiltinMonitorItemPrototype tmp = +// uniqueBuiltinPrototypes.get(rawDataTypeName); + BuiltinMonitorItemPrototype tmp = + uniqueBuiltinPrototypes.get(name); + if (tmp == null || tmp.getPeriod() > prototype.getPeriod()) { + //uniqueBuiltinPrototypes.put(rawDataTypeName, prototype); + uniqueBuiltinPrototypes.put(name, prototype); + } + } + + Iterator> iter2 = + uniqueBuiltinPrototypes.entrySet().iterator(); + while (iter2.hasNext()) { + Map.Entry entry = iter2.next(); + String iterName = entry.getValue().getName(); + MonitorItemCollector collector = this.allCollectors.get(iterName); + this.builtinCollectorTimedScheduler.scheduleAtFixedRate( + collector, rand.nextInt(60000), entry.getValue().getPeriod()); + } + + this.uniqueBuiltinPrototypesByRawData = uniqueBuiltinPrototypes; + } + + private void launchPluginCollectors() { + // TODO + } + + + private void launchMonitorItemCollectors() { + this.initAllCollectors(); + this.launchBuiltinCollectors(); + this.launchPluginCollectors(); + } + + + + private void launchMonitorItemExtractors() { + this.initAllExtractors(); + Iterator>> iter = + this.allExtractors.entrySet().iterator(); + while (iter.hasNext()) { + Entry> entry = iter.next(); + MonitorItemExtractor extractor = entry.getValue(); + this.extractorTimedScheduler.scheduleAtFixedRate( + extractor, 0, (extractionPeriod * 1000)); + } + } + + + @Override + public void run() { + boolean metricCollected = + MonitorManager.getGlobalConf().getBoolean( + "monitor.collection.metric.enable", true); + if (metricCollected) { + new Thread(new CollectionMetricDBDumper()).start(); + try { + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + this.launchMonitorItemCollectors(); + this.launchMonitorItemExtractors(); + } + + /** + * @param extractedDataPool the extractedDataPool to set + */ + public void setExtractedDataPool(Map> extractedDataPool) { + this.extractedDataPool = extractedDataPool; + } + + /** + * @return the extractedDataPool + */ + public Map> getExtractedDataPool() { + return extractedDataPool; + } + + + /** + * @param args + */ + public static void main(String[] args) { + + MonitorItemManager mim = new MonitorItemManager(); + mim.run(); + + for (int i = 0; i < 30; i++) { + + synchronized (mim.rawDataPool) { + System.out.println("--- RAW DATA POOL (" + + mim.rawDataPool.size() +") --- "); + Iterator>> iter = + mim.rawDataPool.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + //String itemName = entry.getKey(); + MonitorItemData itemData = entry.getValue(); + System.out.println("\t---> " + + itemData.getName() + "\t" + + itemData.getData() + "\t" + + new Date(itemData.getTimestamp()) + "\t" + + itemData.getAdditionalInfo()); + } + } + + synchronized (mim.extractedDataPool) { + System.out.println("--- EXT DATA POOL (" + + mim.getExtractedDataPool().size() +") --- "); + Iterator>> iter = + mim.getExtractedDataPool().entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + //String itemName = entry.getKey(); + MonitorItemData itemData = entry.getValue(); + System.out.println("\t---> " + + itemData.getName() + "\t" + + itemData.getData() + "\t" + + new Date(itemData.getTimestamp()) + "\t" + + itemData.getAdditionalInfo()); + } + } + + try { + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + /* + System.out.println("--- RAW DATA POOL (" + mim.rawDataPool.size() +") ---> " + + new Date(mim.rawDataPool.get("hma.monitor.collection.SimMonitorItemRawData").getTimestamp())); + System.out.println("--- EXT DATA POOL (" + mim.getExtractedDataPool().size() +") ---> " + + mim.getExtractedDataPool().get("SimulationMonitorItem").getName() + " " + + mim.getExtractedDataPool().get("SimulationMonitorItem").getData()); + // + new Date(mim.extractedDataPool.get("SimulationMonitorItem").getTimestamp())); + System.out.println("--- EXT DATA POOL (" + mim.getExtractedDataPool().size() +") ---> " + + mim.getExtractedDataPool().get("SimulationMonitorItem2").getName() + " " + + mim.getExtractedDataPool().get("SimulationMonitorItem2").getData()); + System.out.println("--- EXT DATA POOL (" + mim.getExtractedDataPool().size() +") ---> " + + mim.getExtractedDataPool().get("SimulationMonitorItem3").getName() + " " + + mim.getExtractedDataPool().get("SimulationMonitorItem3").getData()); + try { + TimeUnit.SECONDS.sleep(7); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + System.out.println("--- EXT DATA POOL (" + mim.getExtractedDataPool().size() +") ---> " + + mim.getExtractedDataPool().get("NameNode_ListHDFSRootPath").getName() + " " + + mim.getExtractedDataPool().get("NameNode_ListHDFSRootPath").getData()); + */ + } + } + +} diff --git a/src/java/hma/monitor/MonitorManager.java b/src/java/hma/monitor/MonitorManager.java new file mode 100644 index 0000000..56c2c76 --- /dev/null +++ b/src/java/hma/monitor/MonitorManager.java @@ -0,0 +1,313 @@ +/** + * + */ +package hma.monitor; + +import hma.conf.Configuration; +import hma.conf.Configured; +import hma.util.LOG; +import org.apache.hadoop.ipc.ProtocolSignature; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.net.NetUtils; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.concurrent.TimeUnit; + +/** + * @author guoyezhi + */ +public class MonitorManager extends Configured implements HMAMonitorProtocol { + + private static MonitorManager monitorManager = null; + + private static boolean isUpdate = false; + + private Server server; + + private InetSocketAddress address; + + private static boolean isAlarm = true; + + private final long alarmInterval = 60 * 1000 * 60 * 2; // 2 hours + + private MonitorRecover recover = new MonitorRecover(); + + private MonitorManager(Configuration mmConf) { + super(mmConf); + MonitorManager.monitorManager = this; + startMonitorManager(); + } + + private void startMonitorManager() { + MonitorItemManager miManager = + new MonitorItemManager(); + MonitorStrategyManager msManager = + new MonitorStrategyManager(miManager.getExtractedDataPool()); + new Thread(miManager).start(); + + /*try { + createManagerRPC(); + } catch (IOException e) { + e.printStackTrace(); + }*/ + + recover.start(); + + try { + TimeUnit.SECONDS.sleep(120); + } catch (InterruptedException e) { + e.printStackTrace(); + } + new Thread(msManager).start(); + + + } + + public static MonitorManager createMonitorManager( + String args[], + Configuration mmConf) { + + // TODO: utilize args[] in future + System.out.println("Args size : " + args.length); + if (args.length == 2) { + if (args[1].equals("updateamm")) { + System.out.println("Update appmaster monitor ..."); + isUpdate = true; + } + } + + if (MonitorManager.monitorManager == null) { + if (mmConf == null) + mmConf = new Configuration(true, "hma"); + return new MonitorManager(mmConf); + } else { + //LOG.warn("MonitorManager has already been created."); + return MonitorManager.monitorManager; + } + } + + public static MonitorManager getMonitorManager() { + return MonitorManager.monitorManager; + } + + /** + * @return the clusterName + */ + public static String getMonitoredClusterName() { + MonitorManager manager = + MonitorManager.getMonitorManager(); + if (manager == null) + return null; + return manager.getConf().get( + "monitored.hadoop.cluster.id"); + } + + public static Configuration getGlobalConf() { + if (monitorManager == null) + return null; + return monitorManager.getConf(); + } + + public static void startupShutDownMessage() { + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + System.out.println("Shutting down"); + LOG.info("Shutting down"); + System.out.flush(); + System.err.flush(); + + } + }); + } + + /** + * @param args + */ + public static void main(String[] args) throws Exception { + startupShutDownMessage(); + //System.out.println(MonitorManager.class.getResource("MonitorItem.java")); + MonitorManager monitorManager = + MonitorManager.createMonitorManager(args, null); + System.out.println(MonitorManager.getMonitoredClusterName()); + } + + /*private void createManagerRPC() throws IOException { + address = NetUtils.createSocketAddr("127.0.0.1", 0); + server = RPC.getServer(this, address.getHostName(), address.getPort(), 3, false, new org.apache.hadoop.conf.Configuration()); + server.start(); + LOG.info("HMA manager rpc server is started at : " + server.getListenerAddress().getAddress() + ":" + server.getListenerAddress().getPort()); + saveRPCInfo(); + LOG.info("HMA manager rpc server info is saved "); + }*/ + + private void saveRPCInfo() { + File f = new File(System.getProperty("hma.home.dir") + "/conf/" + getMonitoredClusterName().toUpperCase() + "/hma-rpc.prop"); + if (f.exists() && f.isFile()) { + System.out.println(" ^^^ hma-rpc.prop has been deleted ..."); + f.delete(); + } + try { + f.createNewFile(); + BufferedWriter output = new BufferedWriter(new FileWriter(f)); + output.write(getMonitoredClusterName() + ":" + server.getListenerAddress().getAddress().getHostName() + ":" + server.getListenerAddress().getPort() + "\n"); + output.close(); + } catch (IOException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + } + + public static boolean isAlarm() { + return isAlarm; + } + + @Override + public boolean suspendAlarm(String cluster) { + if (getMonitoredClusterName().equals(cluster.toUpperCase())) { + isAlarm = false; + recover.resetTime(); + recover.isRunning = true; + + LOG.warn("Receive suspendAlarm request"); + String message = cluster + "集群短信报警已被关闭，2小时后自动恢复"; + sendSMS(message); + return true; + } + LOG.warn("Receive suspendAlarm request, cluster is not match"); + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + @SuppressWarnings("deprecation") + @Override + public boolean recoverAlarm(String cluster) { + if (getMonitoredClusterName().equals(cluster.toUpperCase())) { + isAlarm = true; + recover.isRunning = false; + + LOG.warn("Receive recoverAlarm request"); + String message = cluster + "集群短信报警已经恢复，请务必Check服务状态"; + sendSMS(message); + return true; + } + LOG.warn("Receive recoverAlarm request, cluster is not match"); + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public MonitorStatus getAlarmStatus(String cluster) { + if (getMonitoredClusterName().equals(cluster.toUpperCase())) { + MonitorStatus s = new MonitorStatus(); + s.setAlarm(isAlarm); + s.setBegin(recover.begin); + s.setEnd(recover.end); + return s; + } + LOG.warn("Receive getAlarmStatus request, cluster is not match"); + return null; //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public long getProtocolVersion(String s, long l) throws IOException { + return 0; //To change body of implemented methods use File | Settings | File Templates. + } + + public ProtocolSignature getProtocolSignature(String s, long l, int i) throws IOException { + return null; + } + + //@Override + public boolean isAuthorized(String s) throws IOException { + return true; //To change body of implemented methods use File | Settings | File Templates. + } + + private void sendSMS(String message) { + String[] servers = + MonitorManager.getGlobalConf().getStrings("baidu.gsm.servers"); + String hmaGSMsend = + System.getProperty("hma.home.dir") + "/conf/" + + MonitorManager.getMonitoredClusterName() + "/gsmsend.hma"; + String[] receivers = + MonitorManager.getGlobalConf().getStrings("sre.mobile.reporter.critical.level.alarm.receivers"); + //String[] receivers = {"18612540350"}; + + Runtime rt = Runtime.getRuntime(); + + for (int i = 0; i < receivers.length; i++) { + + String[] gsmsendComm = new String[3 + 2 * servers.length]; + gsmsendComm[0] = "gsmsend"; + for (int j = 0; j < servers.length; j++) { + gsmsendComm[1 + 2 * j] = "-s"; + gsmsendComm[2 + 2 * j] = servers[j].trim(); + } + gsmsendComm[1 + 2 * servers.length] = receivers[i].trim(); + gsmsendComm[2 + 2 * servers.length] = message; + + for (String str : gsmsendComm) { + System.out.print(str + " "); + } + System.out.println(); + + gsmsendComm[0] = hmaGSMsend; + try { + rt.exec(gsmsendComm); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + + } + } + + + class MonitorRecover extends Thread { + + private long begin; + private long end; + public boolean isRunning = false; + + public MonitorRecover() { + begin = System.currentTimeMillis(); + end = begin + alarmInterval; + } + + public void resetTime() { + begin = System.currentTimeMillis(); + end = begin + alarmInterval; + } + + @Override + public void run() { + while (true) { + if (isRunning) { + if (isAlarm == true) + continue; + long now = System.currentTimeMillis(); + while (now < end && isRunning) { + try { + TimeUnit.SECONDS.sleep(120); + } catch (InterruptedException e) { + e.printStackTrace(); + } + now = System.currentTimeMillis(); + } + if (isRunning) { + isAlarm = true; + LOG.warn("Monitor is recovered ..."); + String message = getMonitoredClusterName() + "集群短信报警已经关闭2小时，自动恢复"; + sendSMS(message); + } + } else { + try { + TimeUnit.SECONDS.sleep(120); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + } +} diff --git a/src/java/hma/monitor/MonitorStatus.java b/src/java/hma/monitor/MonitorStatus.java new file mode 100644 index 0000000..518c0f4 --- /dev/null +++ b/src/java/hma/monitor/MonitorStatus.java @@ -0,0 +1,54 @@ +package hma.monitor; + +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Created with IntelliJ IDEA. + * User: uuleon + * Date: 13-4-28 + * Time: 下午11:04 + * To change this template use File | Settings | File Templates. + */ +public class MonitorStatus implements Writable { + + private boolean isAlarm; + private long begin, end; + + public MonitorStatus() {} + + public void setAlarm(boolean alarm){ + this.isAlarm = alarm; + } + + public boolean getAlarm(){ + return isAlarm; + } + + public void setBegin(long begin) { + this.begin = begin; + } + + public long getBegin(){ return begin;} + + public void setEnd(long end) { this.end = end;} + + public long getEnd() { return end;} + + @Override + public void write(DataOutput out) throws IOException { + out.writeBoolean(isAlarm); + out.writeLong(begin); + out.writeLong(end); + } + + @Override + public void readFields(DataInput in) throws IOException { + this.isAlarm = in.readBoolean(); + this.begin = in.readLong(); + this.end = in.readLong(); + } +} diff --git a/src/java/hma/monitor/MonitorStrategy.java b/src/java/hma/monitor/MonitorStrategy.java new file mode 100644 index 0000000..59429a6 --- /dev/null +++ b/src/java/hma/monitor/MonitorStrategy.java @@ -0,0 +1,134 @@ +/** + * + */ +package hma.monitor; + +import java.util.Date; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; + +import hma.monitor.collection.MonitorItemData; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyPrototype; + +/** + * @author guoyezhi + * + */ +public class MonitorStrategy { + + private MonitorStrategyPrototype prototype = null; + + private Map> dataPool = null; + + private NavigableMap dataProcessingCentre = null; + + public MonitorStrategy(MonitorStrategyPrototype prototype, + Map> dataPool) { + this.setPrototype(prototype); + this.setDataPool(dataPool); + this.setDataProcessingCentre( + new TreeMap()); + } + + /** + * @param prototype the prototype to set + */ + public void setPrototype(MonitorStrategyPrototype prototype) { + this.prototype = prototype; + } + + /** + * @return the prototype + */ + public MonitorStrategyPrototype getPrototype() { + return prototype; + } + + /** + * @param dataPool the dataPool to set + */ + public void setDataPool(Map> dataPool) { + this.dataPool = dataPool; + } + + /** + * @return the dataPool + */ + public Map> getDataPool() { + return dataPool; + } + + /** + * @param dataProcessingCentre the dataProcessingCentre to set + */ + public void setDataProcessingCentre(NavigableMap dataProcessingCentre) { + this.dataProcessingCentre = dataProcessingCentre; + } + + /** + * @return the dataProcessingCentre + */ + public NavigableMap getDataProcessingCentre() { + return dataProcessingCentre; + } + + @SuppressWarnings("unchecked") + public synchronized MonitorStrategyData retrieveSingleCycleStrategyData( + String monitorItemName, Date date) { + return (MonitorStrategyData) this.dataProcessingCentre.get(date).get(monitorItemName); + } + + public synchronized MonitorStrategyDataCollection retrieveSingleCycleStrategyDataCollection( + Date date) { + return this.dataProcessingCentre.get(date); + } + + public synchronized NavigableMap + retrieveMultipleCyclesStrategyDataCollection(Date cutoff, int lastCycles) { + if (lastCycles < 1) + throw new RuntimeException(); + NavigableMap multipleDataCollection = + new TreeMap(); + Date date = cutoff; + for (int i = lastCycles; i > 0 && date != null; i--) { + MonitorStrategyDataCollection dataCollection = + this.dataProcessingCentre.get(date); + multipleDataCollection.put(date, dataCollection); + date = this.dataProcessingCentre.lowerKey(date); + } + if (multipleDataCollection.size() > 0) + return multipleDataCollection; + return null; + } + + @SuppressWarnings("unchecked") + public synchronized NavigableMap> retrieveMultipleCyclesStrategyData( + String monitorItemName, Date cutoff, int lastCycles) { + if (lastCycles < 1) + throw new RuntimeException(); + NavigableMap> multipleData = + new TreeMap>(); + Date date = cutoff; + for (int i = lastCycles; i > 0 && date != null; i--) { + MonitorStrategyData data = + (MonitorStrategyData) this.dataProcessingCentre.get(date).get(monitorItemName); + multipleData.put(date, data); + date = this.dataProcessingCentre.lowerKey(date); + } + if (multipleData.size() != 0) + return multipleData; + return null; + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/monitor/MonitorStrategyManager.java b/src/java/hma/monitor/MonitorStrategyManager.java new file mode 100644 index 0000000..05b69c1 --- /dev/null +++ b/src/java/hma/monitor/MonitorStrategyManager.java @@ -0,0 +1,170 @@ +/** + * + */ +package hma.monitor; + +import hma.conf.Configured; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.strategy.MonitorStrategyConfiguration; +import hma.monitor.strategy.MonitorStrategyPrototype; +import hma.monitor.strategy.MonitorStrategyWorker; +import hma.util.TimedTaskManager; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Random; + +import java.util.Date; +import hma.util.DaStrategyInjector; + +/** + * @author guoyezhi + * + */ +public class MonitorStrategyManager extends Configured implements Runnable { + + private Map> extractedDataPool = null; + + private Map allStrategyPrototypes = null; + + private Map allStrategies = null; + + private Map allStrategyWorkers = null; + + private TimedTaskManager strategyWorkerTimedScheduler = null; + + private static final int MAX_MONITOR_STRATEGY_WORKERS = 40; + + private Random rand = null; + + + /** + * + * @param extractedDataPool + */ + public MonitorStrategyManager(Map> extractedDataPool) { + super(new MonitorStrategyConfiguration(true)); + this.extractedDataPool = extractedDataPool; + this.strategyWorkerTimedScheduler = + new TimedTaskManager(MAX_MONITOR_STRATEGY_WORKERS); + this.rand = new Random(); + } + + private synchronized void initStrategyPrototypes() { + if (allStrategyPrototypes == null) { + MonitorStrategyConfiguration msConf = + (MonitorStrategyConfiguration) this.getConf(); + allStrategyPrototypes = msConf.getAllPrototypes(); + DaStrategyInjector.DaStrategyInjector(allStrategyPrototypes); + } + //System.out.println("!!!---> " + allStrategyPrototypes.size() + // + "Strategy Prototype(s) <---!!!"); + } + + private synchronized void initAllStrategies() { + if (allStrategies == null) { + + if (allStrategyPrototypes == null) { + initStrategyPrototypes(); + if (allStrategyPrototypes == null) { + throw new RuntimeException( + "Invaild Monitor Strategy Conf"); + } + } + + allStrategies = Collections.synchronizedMap( + new HashMap()); + Iterator> iter = + this.allStrategyPrototypes.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = + iter.next(); + String strategyName = entry.getKey(); + MonitorStrategyPrototype prototype = entry.getValue(); + MonitorStrategy strategy = new MonitorStrategy( + prototype, this.extractedDataPool); + this.allStrategies.put(strategyName, strategy); + } + if (allStrategies.size() == 0) { + allStrategies = null; + } + } + } + + private synchronized void initAllStrategyWorkers() { + if (allStrategyWorkers == null) { + + if (allStrategies == null) { + initAllStrategies(); + if (allStrategies == null) { + throw new RuntimeException( + "Invaild Monitor Strategy Conf"); + } + } + + allStrategyWorkers = Collections.synchronizedMap( + new HashMap()); + Iterator> iter = + this.allStrategies.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = + iter.next(); + String strategyName = entry.getKey(); + MonitorStrategy strategy = entry.getValue(); + MonitorStrategyWorker worker; + try { + worker = new MonitorStrategyWorker(strategy); + //for (Date d:worker.getStrategy().getDataProcessingCentre().keySet()){ + // System.out.println("kain's log see wether my item in : " + worker.getStrategy().getDataProcessingCentre().get(d).getInvolvedMonitorItemNames()); + // } + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new RuntimeException(e); + } + allStrategyWorkers.put(strategyName, worker); + } + if (allStrategyWorkers.size() == 0) { + allStrategyWorkers = null; + } + } + } + + private synchronized void launchMonitorStrategyWorkers() { + if (allStrategyWorkers == null) { + this.initAllStrategyWorkers(); + if (allStrategyWorkers == null) { + throw new RuntimeException("Invaild Monitor Strategy Conf"); + } + } + Iterator> iter = + this.allStrategyWorkers.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + MonitorStrategyWorker worker = entry.getValue(); + System.out.println("kain's log to see strategy : " + entry.getKey() + " " + entry.getValue().toString()); + strategyWorkerTimedScheduler.scheduleAtFixedRate( + worker, rand.nextInt(60000), 60000); + } + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + this.launchMonitorStrategyWorkers(); + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/monitor/collection/BuiltinMonitorItemPrototype.java b/src/java/hma/monitor/collection/BuiltinMonitorItemPrototype.java new file mode 100644 index 0000000..4b5773d --- /dev/null +++ b/src/java/hma/monitor/collection/BuiltinMonitorItemPrototype.java @@ -0,0 +1,38 @@ +/** + * + */ +package hma.monitor.collection; + +/** + * @author guoyezhi + * + */ +public class BuiltinMonitorItemPrototype extends MonitorItemPrototype { + + private Class workerClass = null; + + public BuiltinMonitorItemPrototype( + String name, + String type, + long period, + String description, + String className, + String rawDataTypeName) { + super(name, type, period, description, rawDataTypeName); + this.setMonitorItemWorkerClass(className); + } + + public void setMonitorItemWorkerClass(String className) { + try { + workerClass = Class.forName(className); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + public Class getMonitorItemWorkerClass() { + return workerClass; + } + +} diff --git a/src/java/hma/monitor/collection/MonitorItemConfiguration.java b/src/java/hma/monitor/collection/MonitorItemConfiguration.java new file mode 100644 index 0000000..e61d816 --- /dev/null +++ b/src/java/hma/monitor/collection/MonitorItemConfiguration.java @@ -0,0 +1,169 @@ +/** + * + */ +package hma.monitor.collection; + +import hma.conf.Configuration; +import hma.util.LOG; +import hma.util.StringHelper; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; + +/** + * @author Vergil + * + */ +public class MonitorItemConfiguration extends Configuration { + + private HashMap properties = null; + + public MonitorItemConfiguration() { + this(true); + } + + /* + public MonitorItemConfiguration(boolean loadDefaults) { + super(false); + // this.setPropertyValueTagString("type"); + List resources = this.getResources(); + if (loadDefaults) { + resources.add("D:\\eclipse\\guoyezhi\\workspace\\HMA\\conf\\monitor-items-default.xml"); + //resources.add("D:\\eclipse\\guoyezhi\\workspace\\HMA\\src\\monitor-items-site.xml"); + this.setResources(resources); + } + // LOG.debug(resources.toString()); + } + */ + + /* + * TODO: + */ + public MonitorItemConfiguration(boolean loadDefaults) { + super(loadDefaults, "monitor-items"); + } + + + public MonitorItemConfiguration(boolean loadDefaults, String moduleName) { + super(loadDefaults, moduleName); + } + + public synchronized HashMap getAllPrototypes() { + if (properties == null) { + properties = getProps(); + } + return new HashMap(properties); // return a copy + } + + public synchronized HashMap getBuiltinPrototypes() { + if (properties == null) { + properties = getProps(); + } + HashMap biProperties = + new HashMap(); + Iterator> iter = + properties.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String name = entry.getKey(); + MonitorItemPrototype prototype = entry.getValue(); + if (prototype.getType().equals( + MonitorItemPrototype.MONITOR_ITEM_TYPE_BUILTIN)) { + biProperties.put(name, (BuiltinMonitorItemPrototype) prototype); + } + } + return biProperties; // return a copy + } + + private synchronized HashMap getProps() { + + List list = null; + + List resources = this.getResources(); + boolean quietmode = this.getQuietMode(); + + if (properties == null) { + properties = new HashMap(); + list = loadResources(resources, quietmode); + + Iterator iter = list.iterator(); + while(iter.hasNext()) { + Element prop = iter.next(); + if (!"monitor_item".equals(prop.getTagName())) { + // LOG.warn("bad monitor item conf file: element not "); + continue; + } + + NodeList fields = prop.getChildNodes(); + String name = null; + String type = null; + String worker = null; + String rawdata = null; + String period = null; + String description = null; + // TODO! add plug-in property sub-tag + for (int i = 0; i < fields.getLength(); i++) { + Node fieldNode = fields.item(i); + if (!(fieldNode instanceof Element)) + continue; + Element field = (Element)fieldNode; + if ("name".equals(field.getTagName()) && field.hasChildNodes()) + name = ((Text)field.getFirstChild()).getData().trim(); + if ("type".equals(field.getTagName()) && field.hasChildNodes()) + type = ((Text)field.getFirstChild()).getData().trim(); + if ("period".equals(field.getTagName()) && field.hasChildNodes()) + period = ((Text)field.getFirstChild()).getData().trim(); + if ("description".equals(field.getTagName()) && field.hasChildNodes()) + description = ((Text)field.getFirstChild()).getData().trim(); + if ("worker".equals(field.getTagName()) && field.hasChildNodes()) + worker = ((Text)field.getFirstChild()).getData().trim(); + if ("rawdata".equals(field.getTagName()) && field.hasChildNodes()) + rawdata = ((Text)field.getFirstChild()).getData().trim(); + } + + if (name != null && type != null && worker != null + && rawdata != null && period != null) { + description = StringHelper.removeDuplicateWhitespace(description); + LOG.info(name + "\n" + type + "\n" + worker + "\n" + rawdata + "\n" + + period + "\n" + description); + MonitorItemPrototype mip = null; + if (type.equals(MonitorItemPrototype.MONITOR_ITEM_TYPE_BUILTIN)) { + mip = new BuiltinMonitorItemPrototype(name, type, + Long.parseLong(period), description, worker, rawdata); + } else if (type.equals(MonitorItemPrototype.MONITOR_ITEM_TYPE_PLUGIN)) { + + } else { + throw new RuntimeException("wrong monitor item type configured"); + } + + properties.put(name, mip); + } + } + } + + return properties; + } + + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + MonitorItemConfiguration conf = new MonitorItemConfiguration(); + LOG.info("conf.get(\"SimulationMonitorItem\") = " + + conf.get("SimulationMonitorItem")); + LOG.info("conf.get(\"SimulationMonitorItem\") = " + + conf.get("hadoop.job.history.cleandays")); + //conf.getProps().list(System.out); + LOG.info(conf.getProps().toString()); + } + +} diff --git a/src/java/hma/monitor/collection/MonitorItemData.java b/src/java/hma/monitor/collection/MonitorItemData.java new file mode 100644 index 0000000..c93f0cd --- /dev/null +++ b/src/java/hma/monitor/collection/MonitorItemData.java @@ -0,0 +1,78 @@ +/** + * + */ +package hma.monitor.collection; + +/** + * This class represents a monitor item data (both raw type and extracted type). + * The difference between a monitor item data's raw-type MonitorItemData object + * and its extracted-type MonitorItemData object is that data properties + * of these two objects are probably different. That is, name and + * timestamp properties of these two objects should be consistent. + * + * @author guoyezhi + * add additionalInfo by chiwen01 + * + */ +public class MonitorItemData { + + private String name; + + private TYPE data = null; + + private long timestamp = 0; + + private String additionalInfo = null; + + public MonitorItemData(String name, TYPE data, long timestamp) { + this.setName(name); + this.data = data; + this.timestamp = timestamp; + } + + public MonitorItemData(String name, TYPE data, long timestamp, String additionalInfo) { + this.setName(name); + this.data = data; + this.timestamp = timestamp; + this.additionalInfo = additionalInfo; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + public TYPE getData() { + return data; + } + + public void setData(TYPE data) { + this.data = data; + } + + public long getTimestamp() { + return timestamp; + } + + public String getAdditionalInfo() { + return additionalInfo; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public void setAdditionalInfo(String additionalInfo) { + this.additionalInfo = additionalInfo; + } + +} diff --git a/src/java/hma/monitor/collection/MonitorItemPrototype.java b/src/java/hma/monitor/collection/MonitorItemPrototype.java new file mode 100644 index 0000000..2db355b --- /dev/null +++ b/src/java/hma/monitor/collection/MonitorItemPrototype.java @@ -0,0 +1,109 @@ +/** + * + */ +package hma.monitor.collection; + +/** + * @author guoyezhi + * + */ +public abstract class MonitorItemPrototype { + + private String name = null; + private String type = null; + private long period = 180000; // 180sec by default? + private String description = null; + private String rawDataTypeName = null; + + public static final String MONITOR_ITEM_TYPE_BUILTIN = "built-in"; + public static final String MONITOR_ITEM_TYPE_PLUGIN = "plug-in"; + + public MonitorItemPrototype(String name, String type, long period, + String description) { + this(name, type, period, description, " " + name); + } + + public MonitorItemPrototype(String name, String type, long period, + String description, String rawDataTypeName) { + this.setName(name); + this.setType(type); + this.setPeriod(period); + this.setDescription(description); + this.setRawDataTypeName(rawDataTypeName); + } + + /** + * @param name + * the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param type + * the type to set + */ + public void setType(String type) { + this.type = type; + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @param period + * the period to set + */ + public void setPeriod(long period) { + this.period = period; + } + + /** + * @return the period + */ + public long getPeriod() { + return period; + } + + /** + * @param description + * the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param rawDataTypeName the rawDataTypeName to set + */ + public void setRawDataTypeName(String rawDataTypeName) { + this.rawDataTypeName = rawDataTypeName; + } + + /** + * @return the rawDataTypeName + */ + public String getRawDataTypeName() { + return rawDataTypeName; + } + +} \ No newline at end of file diff --git a/src/java/hma/monitor/collection/MonitorItemWorker.java b/src/java/hma/monitor/collection/MonitorItemWorker.java new file mode 100644 index 0000000..5611593 --- /dev/null +++ b/src/java/hma/monitor/collection/MonitorItemWorker.java @@ -0,0 +1,28 @@ +/** + * + */ +package hma.monitor.collection; + +/** + * @author guoyezhi + * + */ +public abstract class MonitorItemWorker { + + private MonitorItemPrototype prototype = null; + + public abstract MonitorItemData collect(); + + public abstract MonitorItemData extract(MonitorItemData rawData); + + public void setMonitorItemPrototype(MonitorItemPrototype prototype) { + this.prototype = prototype; + } + + public MonitorItemPrototype getMonitorItemPrototype() { + return this.prototype; + } + + public String MyItemName = null; + +} diff --git a/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker.java b/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker.java new file mode 100644 index 0000000..2349eef --- /dev/null +++ b/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker.java @@ -0,0 +1,194 @@ +/** + * + */ +package hma.monitor.collection.custom; + +import hma.monitor.MonitorManager; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemWorker; +import hma.monitor.collection.metric.CollectionMetricPool; +import hma.monitor.collection.metric.CollectionMetricRecord; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * @author guoyezhi + * + 通过rpc方式，调用copyfromlocal上传一个文件再删除，监控其可写性 + * + */ +public class HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker extends MonitorItemWorker< + Map, + Map> { + + public static final String className = + HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker.class.getSimpleName(); + + public static final Log LOG = + LogFactory.getLog(HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker.class); + + public static final String MONITOR_ITEM_NAME = + "HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark"; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private static SimpleDateFormat fileIDDateFormat = + new SimpleDateFormat("yyyyMMddHHmmss"); + + private Configuration hadoopConf = null; + + + public HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker() { + String hadoopConfDir = + MonitorManager.getMonitorManager().getConf().get( + "hadoop.client.conf.dir"); + hadoopConf = new Configuration(false); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-default.xml")); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-site.xml")); + } + + + public static enum BenchmarkOP { + CopyFromLocal, CopyToLocal, Remove, RemoveRecursively + } + + + @Override + public MonitorItemData> collect() { + + Map testResult = new HashMap(); + + FileSystem fs = null; + StringBuilder sb = new StringBuilder(); + long uploadBegTimestamp = 0; + long uploadEndTimestamp = 0; + long removeBegTimestamp = 0; + long removeEndTimestamp = 0; + + try { + sb.append("\n\t[" + dateFormat.format(new Date()) + " - " + + className + "] Getting DistributedFileSystem Proxy ..."); + fs = FileSystem.get(hadoopConf); + sb.append("\n\t[" + dateFormat.format(new Date()) + " - " + + className + "] Have got DistributedFileSystem Proxy"); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + + Path src = new Path( + System.getProperty("hma.home.dir") + "/benchmark/hdfs.copyfromlocal.benchmark"); + System.out.println("kain's log about FileUpload src" + src); + uploadBegTimestamp = System.currentTimeMillis(); + Path dst = new Path( + "/monitor/hma4hdfs/test." + fileIDDateFormat.format(new Date(uploadBegTimestamp))); + System.out.println("kain's log about FileUpload dst" + dst); + try { + + sb.append("\n\t[" + dateFormat.format(new Date(uploadBegTimestamp)) + " - " + + className + "] Uploading benchmark file to " + dst + " ..."); + fs.copyFromLocalFile(false, false, src, dst); + uploadEndTimestamp = System.currentTimeMillis(); + sb.append("\n\t[" + dateFormat.format(new Date(uploadEndTimestamp)) + " - " + + className + "] Have uploaded benchmark file to " + dst); + + System.out.println("kain's log about FileUpload sb" + sb.toString()); + + testResult.put(BenchmarkOP.CopyFromLocal, uploadEndTimestamp - uploadBegTimestamp); + + CollectionMetricPool.offerRecord( + new CollectionMetricRecord( + MonitorManager.getMonitoredClusterName(), + "HDFS_RPC_FileSystem_UploadBenchmarkFile", + uploadBegTimestamp, + uploadEndTimestamp - uploadBegTimestamp)); + + try { + TimeUnit.MINUTES.sleep(5); + } catch (InterruptedException e) { + e.printStackTrace(); + } +// +// removeBegTimestamp = System.currentTimeMillis(); +// sb.append("\n\t[" + dateFormat.format(new Date(removeBegTimestamp)) + " - " +// + className + "] Removing benchmark file " + dst + " ..."); +// boolean res = fs.delete(dst, false); +// removeEndTimestamp = System.currentTimeMillis(); +// if (res) { +// sb.append("\n\t[" + dateFormat.format(new Date(removeEndTimestamp)) + " - " +// + className + "] Have removed benchmark file " + dst); +// } +// +// System.out.println("kain's log about FileUpload remove sb" + sb.toString()); +// testResult.put(BenchmarkOP.Remove, removeEndTimestamp - removeBegTimestamp); +// System.out.println("kain's log about FileUpload testResult" + testResult.toString()); +// +// CollectionMetricPool.offerRecord( +// new CollectionMetricRecord( +// MonitorManager.getMonitoredClusterName(), +// "HDFS_RPC_FileSystem_RemoveBenchmarkFile", +// removeBegTimestamp, +// removeEndTimestamp - removeBegTimestamp)); + + } + catch (IOException e) { + e.printStackTrace(); + return null; + } + finally { + /* + * Notice: + * Should not close DistributedFileSystem which + * is created by invoking static method + * FileSystem.get(Configuration), otherwise the + * other threads which are using DistributedFileSystem + * will go wrong! + * + * if (fs != null) { + * try { + * fs.close(); + * } catch (IOException e) { + * e.printStackTrace(); + * } + * } + */ + } + + LOG.info(sb.toString()); + + return new MonitorItemData>( + MONITOR_ITEM_NAME, testResult, uploadBegTimestamp); + + } + + @Override + public MonitorItemData> extract( + MonitorItemData> rawData) { + return rawData; + } + + + /** + * for unit testing + * + * @param args + */ + public static void main(String[] args) { + HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker worker = + new HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker(); + worker.extract(worker.collect()); + } + +} + diff --git a/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_ListRootPathEntries_MonitorItemWorker.java b/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_ListRootPathEntries_MonitorItemWorker.java new file mode 100644 index 0000000..7911547 --- /dev/null +++ b/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_ListRootPathEntries_MonitorItemWorker.java @@ -0,0 +1,166 @@ +/** + * + */ +package hma.monitor.collection.custom; + +import hma.monitor.MonitorManager; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemWorker; +import hma.monitor.collection.metric.CollectionMetricPool; +import hma.monitor.collection.metric.CollectionMetricRecord; +import hma.util.EasyWebGet; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +/** + * @author guoyezhi + * + */ +public class HDFS_RPC_FileSystem_ListRootPathEntries_MonitorItemWorker extends + MonitorItemWorker { + + public static final String className = + HDFS_RPC_FileSystem_ListRootPathEntries_MonitorItemWorker.class.getSimpleName(); + + public static final Log LOG = + LogFactory.getLog(HDFS_RPC_FileSystem_ListRootPathEntries_MonitorItemWorker.class); + + public static final String MONITOR_ITEM_NAME = + "HDFS_RPC_FileSystem_ListRootPathEntries"; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private Configuration hadoopConf = null; + + + public HDFS_RPC_FileSystem_ListRootPathEntries_MonitorItemWorker() { + String hadoopConfDir = + MonitorManager.getMonitorManager().getConf().get( + "hadoop.client.conf.dir"); + hadoopConf = new Configuration(false); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-default.xml")); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-site.xml")); + } + + + /* (non-Javadoc) + * @see hma.monitor.collection.MonitorItemWorker#collect() + */ + @Override + public MonitorItemData collect() { + + FileSystem fs = null; + StringBuilder sb = new StringBuilder(); + long begTimestamp = 0; + long acquisitionTimestamp = 0; + long endTimestamp = 0; + StringBuilder keyBuilder = new StringBuilder(); + StringBuilder additionalInfoBuilder = new StringBuilder(); + + try { + begTimestamp = System.currentTimeMillis(); + sb.append("\n\t[" + dateFormat.format(new Date(begTimestamp)) + " - " + + className + "] Getting DistributedFileSystem Proxy ..."); + fs = FileSystem.get(hadoopConf); + sb.append("\n\t[" + dateFormat.format(new Date()) + " - " + + className + "] Have got DistributedFileSystem Proxy"); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + //System.out.println("kain's log"+fs.getWorkingDirectory()); + StringBuffer buffer = new StringBuffer(); + try { + acquisitionTimestamp = System.currentTimeMillis(); + sb.append("\n\t[" + dateFormat.format(new Date(acquisitionTimestamp)) + " - " + + className + "] Getting " + MONITOR_ITEM_NAME + " ..."); + //System.out.println("kain's log start") ; + FileStatus[] statusArray = fs.listStatus(new Path("/")); + //System.out.println("kain's log end fs list") ; + endTimestamp = System.currentTimeMillis(); + sb.append("\n\t[" + dateFormat.format(new Date(endTimestamp)) + " - " + + className + "] Have got " + MONITOR_ITEM_NAME); + + buffer.append("Found " + statusArray.length + " items:\n"); + for (FileStatus status : statusArray) { + buffer.append(status.getPermission() + "\t" + + status.getReplication() + "\t" + + status.getOwner() + "\t" + + status.getGroup() + "\t" + + status.getLen() + "\t" + + new Date(status.getModificationTime()) + "\t" + + status.getPath().toUri().getPath() + "\n"); + } + //System.out.println("kain's log " + buffer.toString()) ; + + } catch (IOException e) { + e.printStackTrace(); + return null; + } finally { + /* + * Notice: + * Should not close DistributedFileSystem which + * is created by invoking static method + * FileSystem.get(Configuration), otherwise the + * other threads which are using DistributedFileSystem + * will go wrong! + * + * if (fs != null) { + * try { + * fs.close(); + * } catch (IOException e) { + * e.printStackTrace(); + * } + * } + */ + } + + CollectionMetricPool.offerRecord( + new CollectionMetricRecord( + MonitorManager.getMonitoredClusterName(), + MONITOR_ITEM_NAME, + begTimestamp, + endTimestamp - begTimestamp)); + + LOG.info(sb.toString()); + + return new MonitorItemData( + MONITOR_ITEM_NAME, buffer.toString(), acquisitionTimestamp); + } + + /* (non-Javadoc) + * @see hma.monitor.collection.MonitorItemWorker#extract(hma.monitor.collection.MonitorItemData) + */ + @Override + public MonitorItemData extract(MonitorItemData rawData) { + return rawData; + } + + + /** + * for unit testing + * + * @param args + */ + public static void main(String[] args) { + + HDFS_RPC_FileSystem_ListRootPathEntries_MonitorItemWorker worker = + new HDFS_RPC_FileSystem_ListRootPathEntries_MonitorItemWorker(); + MonitorItemData data = worker.extract(worker.collect()); + System.out.println(data.getName() + ": <" + + new Date(data.getTimestamp()) + ">"); + System.out.println(data.getData()); + + } +} + diff --git a/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.java b/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.java new file mode 100644 index 0000000..c162ead --- /dev/null +++ b/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.java @@ -0,0 +1,273 @@ +package hma.monitor.collection.custom; + +import hma.monitor.MonitorManager; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemWorker; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * + */ +public class HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker extends MonitorItemWorker< + Map, + Map> { + + public static final String className = + HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.class.getSimpleName(); + + public static final Log LOG = + LogFactory.getLog(HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.class); + + public static final String MONITOR_ITEM_NAME = + "HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm"; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private static SimpleDateFormat fileIDDateFormat = + new SimpleDateFormat("yyyyMMddHHmmss"); + + private Configuration hadoopConf = null; + + + public HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker() { + String hadoopConfDir = + MonitorManager.getMonitorManager().getConf().get( + "hadoop.client.conf.dir"); + hadoopConf = new Configuration(false); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-default.xml")); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-site.xml")); + } + + + public static enum BenchmarkOP { + mkdir, putFile, catFile, rmAllB + } + + + @Override + public MonitorItemData> collect() { + Map resMap = new HashMap(); + Map testResult = new HashMap(); + + FileSystem fs = null; + StringBuilder sb = new StringBuilder(); + long uploadBegTimestamp = 0; + long uploadEndTimestamp = 0; + long removeBegTimestamp = 0; + long removeEndTimestamp = 0; + boolean mkdirB = true; + boolean putFileB = true; + boolean catFileB = true; + boolean rmAllB = true; + + try { + sb.append("\n\t[" + dateFormat.format(new Date()) + " - " + + className + "] Getting DistributedFileSystem Proxy ..."); + fs = FileSystem.get(hadoopConf); + sb.append("\n\t[" + dateFormat.format(new Date()) + " - " + + className + "] Have got DistributedFileSystem Proxy"); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + + Path src = new Path( + System.getProperty("hma.home.dir") + "/benchmark/hdfs.copyfromlocal.benchmark"); + uploadBegTimestamp = System.currentTimeMillis(); + + + Path dstDir = new Path("/monitor/hma4hdfs/testDir"); + + Path dst = new Path( + "/monitor/hma4hdfs/testDir/putTestFile" ); + + Set helpSet = new HashSet(); + + + try { + // 0. init remove dir + + System.out.println("kain's log init rm dstDir"); + boolean res = fs.delete(dstDir, true); + System.out.println("init rm res is " + res); + + //1 . mkdir + fs.mkdirs(dstDir); + for(FileStatus fileS:fs.listStatus(new Path("/monitor/hma4hdfs/"))){ + String dirName = fileS.getPath().toUri().getPath(); + System.out.println("kain's log print dir " + dirName); + helpSet.add(dirName); + if(dirName.equals("/monitor/hma4hdfs/testDir")){ + //System.out.println("kain's log in equal"); + break; + } + } + if(!helpSet.contains("/monitor/hma4hdfs/testDir")){ + mkdirB = false; + resMap.put(BenchmarkOP.mkdir, mkdirB); + return new MonitorItemData>( + MONITOR_ITEM_NAME, resMap, uploadBegTimestamp); + } + + try { + TimeUnit.SECONDS.sleep(8); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + + // 2. put file + helpSet = new HashSet(); + sb.append("\n\t[" + dateFormat.format(new Date(uploadBegTimestamp)) + " - " + + className + "] Uploading benchmark file to " + dst + " ..."); + + fs.copyFromLocalFile(false, false, src, dst); + + uploadEndTimestamp = System.currentTimeMillis(); + sb.append("\n\t[" + dateFormat.format(new Date(uploadEndTimestamp)) + " - " + + className + "] Have uploaded benchmark file to " + dst); + + for(FileStatus fileS:fs.listStatus(new Path("/monitor/hma4hdfs/testDir"))){ + String fileName = fileS.getPath().toUri().getPath(); + System.out.println("kain's log print file name " + fileName); + + helpSet.add(fileName); + if(fileName.equals("/monitor/hma4hdfs/testDir/putTestFile")){ + //System.out.println("kain's log in file equal"); + break; + } + } + + if(!helpSet.contains("/monitor/hma4hdfs/testDir/putTestFile")){ + putFileB = false; + } + System.out.println("kain's log putFileB is " + putFileB); + + try { + TimeUnit.SECONDS.sleep(3); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + + // 3. cat file + if(putFileB){ + System.out.println("kain's log 3rd cat step "); + FSDataInputStream fdis = fs.open(dst); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + IOUtils.copyBytes(fdis, baos, hadoopConf, false); + } finally { + fdis.close(); + } + String catRes = baos.toString(); + System.out.println("kain's log cat file content : " + catRes + + " size is " + catRes.length()); + if(!catRes.contains("hma4hdfs")){ + catFileB = false; + } + + } + + + try { + TimeUnit.SECONDS.sleep(8); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + // 4. remove dir + System.out.println("kain's log to 4th step rm "); + removeBegTimestamp = System.currentTimeMillis(); + sb.append("\n\t[" + dateFormat.format(new Date(removeBegTimestamp)) + " - " + + className + "] Removing benchmark Dir " + dstDir + " ..."); + res = fs.delete(dstDir, true); + removeEndTimestamp = System.currentTimeMillis(); + for(FileStatus fileS:fs.listStatus(new Path("/monitor/hma4hdfs/"))){ + String dirName = fileS.getPath().toUri().getPath(); + if(dirName.equals("/monitor/hma4hdfs/testDir")){ + rmAllB=false; + break; + } + } + + + + if (rmAllB) { + sb.append("\n\t[" + dateFormat.format(new Date(removeEndTimestamp)) + " - " + + className + "] Have removed benchmark file " + dst); + } + resMap.put(BenchmarkOP.mkdir, mkdirB); + resMap.put(BenchmarkOP.putFile, putFileB); + resMap.put(BenchmarkOP.catFile, catFileB); + resMap.put(BenchmarkOP.rmAllB, rmAllB); + System.out.println("kain's log m + p + c + r " + mkdirB + " " + + putFileB + " " + catFileB + " " + rmAllB); + + + } catch (IOException e) { + e.printStackTrace(); + //return null; + } finally { + /* + * Notice: + * Should not close DistributedFileSystem which + * is created by invoking static method + * FileSystem.get(Configuration), otherwise the + * other threads which are using DistributedFileSystem + * will go wrong! + * + * if (fs != null) { + * try { + * fs.close(); + * } catch (IOException e) { + * e.printStackTrace(); + * } + * } + */ + } + + LOG.info(sb.toString()); + + return new MonitorItemData>( + MONITOR_ITEM_NAME, resMap, uploadBegTimestamp); + + } + + @Override + public MonitorItemData> extract(MonitorItemData> rawData) { + return rawData; + } + + + + + + /** + * for unit testing + * + * @param args + */ + public static void main(String[] args) { + /*HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker worker = + new HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker(); + worker.extract(worker.collect());*/ + + } + +} + diff --git a/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.java.bak b/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.java.bak new file mode 100644 index 0000000..8f0d4d2 --- /dev/null +++ b/src/java/hma/monitor/collection/custom/HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.java.bak @@ -0,0 +1,276 @@ +/** + * + */ +package hma.monitor.collection.custom; + +import hma.monitor.MonitorManager; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemWorker; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.io.IOUtils; + +/** + * @author kain + * + */ +public class HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker extends MonitorItemWorker< + Map, + Map> { + + public static final String className = + HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.class.getSimpleName(); + + public static final Log LOG = + LogFactory.getLog(HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.class); + + public static final String MONITOR_ITEM_NAME = + "DFS_RPC_FileSystem_MkdirAndPutAndCatAndRm"; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private static SimpleDateFormat fileIDDateFormat = + new SimpleDateFormat("yyyyMMddHHmmss"); + + private Configuration hadoopConf = null; + + + public HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker() { + String hadoopConfDir = + MonitorManager.getMonitorManager().getConf().get( + "hadoop.client.conf.dir"); + hadoopConf = new Configuration(false); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-default.xml")); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-site.xml")); + } + + + public static enum BenchmarkOP { + mkdir, putFile, catFile, rmAllB + } + + + @Override + public MonitorItemData> collect() { + Map resMap = new HashMap(); + Map testResult = new HashMap(); + + FileSystem fs = null; + StringBuilder sb = new StringBuilder(); + long uploadBegTimestamp = 0; + long uploadEndTimestamp = 0; + long removeBegTimestamp = 0; + long removeEndTimestamp = 0; + boolean mkdirB = true; + boolean putFileB = true; + boolean catFileB = true; + boolean rmAllB = true; + + try { + sb.append("\n\t[" + dateFormat.format(new Date()) + " - " + + className + "] Getting DistributedFileSystem Proxy ..."); + fs = FileSystem.get(hadoopConf); + sb.append("\n\t[" + dateFormat.format(new Date()) + " - " + + className + "] Have got DistributedFileSystem Proxy"); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + + Path src = new Path( + System.getProperty("hma.home.dir") + "/benchmark/hdfs.copyfromlocal.benchmark"); + uploadBegTimestamp = System.currentTimeMillis(); + + + Path dstDir = new Path("/monitor/hma4hdfs/testDir"); + + Path dst = new Path( + "/monitor/hma4hdfs/testDir/putTestFile" ); + + Set helpSet = new HashSet(); + + + try { + // 0. init remove dir + + System.out.println("kain's log init rm dstDir"); + boolean res = fs.delete(dstDir, true); + System.out.println("init rm res is " + res); + + //1 . mkdir + fs.mkdirs(dstDir); + for(FileStatus fileS:fs.listStatus(new Path("/monitor/hma4hdfs/"))){ + String dirName = fileS.getPath().toUri().getPath(); + System.out.println("kain's log print dir " + dirName); + helpSet.add(dirName); + if(dirName.equals("/monitor/hma4hdfs/testDir")){ + //System.out.println("kain's log in equal"); + break; + } + } + if(!helpSet.contains("/monitor/hma4hdfs/testDir")){ + mkdirB = false; + resMap.put(BenchmarkOP.mkdir, mkdirB); + return new MonitorItemData>( + MONITOR_ITEM_NAME, resMap, uploadBegTimestamp); + } + + try { + TimeUnit.SECONDS.sleep(8); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + + // 2. put file + helpSet = new HashSet(); + sb.append("\n\t[" + dateFormat.format(new Date(uploadBegTimestamp)) + " - " + + className + "] Uploading benchmark file to " + dst + " ..."); + + fs.copyFromLocalFile(false, false, src, dst); + + uploadEndTimestamp = System.currentTimeMillis(); + sb.append("\n\t[" + dateFormat.format(new Date(uploadEndTimestamp)) + " - " + + className + "] Have uploaded benchmark file to " + dst); + + for(FileStatus fileS:fs.listStatus(new Path("/monitor/hma4hdfs/testDir"))){ + String fileName = fileS.getPath().toUri().getPath(); + System.out.println("kain's log print file name " + fileName); + + helpSet.add(fileName); + if(fileName.equals("/monitor/hma4hdfs/testDir/putTestFile")){ + //System.out.println("kain's log in file equal"); + break; + } + } + + if(!helpSet.contains("/monitor/hma4hdfs/testDir/putTestFile")){ + putFileB = false; + } + System.out.println("kain's log putFileB is " + putFileB); + + try { + TimeUnit.SECONDS.sleep(3); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + + // 3. cat file + if(putFileB){ + System.out.println("kain's log 3rd cat step "); + FSDataInputStream fdis = fs.open(dst); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + IOUtils.copyBytes(fdis, baos, hadoopConf, false); + } finally { + fdis.close(); + } + String catRes = baos.toString(); + System.out.println("kain's log cat file content : " + catRes + + " size is " + catRes.length()); + if(!catRes.contains("hma4hdfs")){ + catFileB = false; + } + + } + + + try { + TimeUnit.SECONDS.sleep(8); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + // 4. remove dir + System.out.println("kain's log to 4th step rm "); + removeBegTimestamp = System.currentTimeMillis(); + sb.append("\n\t[" + dateFormat.format(new Date(removeBegTimestamp)) + " - " + + className + "] Removing benchmark Dir " + dstDir + " ..."); + res = fs.delete(dstDir, true); + removeEndTimestamp = System.currentTimeMillis(); + for(FileStatus fileS:fs.listStatus(new Path("/monitor/hma4hdfs/"))){ + String dirName = fileS.getPath().toUri().getPath(); + if(dirName.equals("/monitor/hma4hdfs/testDir")){ + rmAllB=false; + break; + } + } + + + + if (rmAllB) { + sb.append("\n\t[" + dateFormat.format(new Date(removeEndTimestamp)) + " - " + + className + "] Have removed benchmark file " + dst); + } + resMap.put(BenchmarkOP.mkdir, mkdirB); + resMap.put(BenchmarkOP.putFile, putFileB); + resMap.put(BenchmarkOP.catFile, catFileB); + resMap.put(BenchmarkOP.rmAllB, rmAllB); + System.out.println("kain's log m + p + c + r " + mkdirB + " " + + putFileB + " " + catFileB + " " + rmAllB); + + + } catch (IOException e) { + e.printStackTrace(); + //return null; + } finally { + /* + * Notice: + * Should not close DistributedFileSystem which + * is created by invoking static method + * FileSystem.get(Configuration), otherwise the + * other threads which are using DistributedFileSystem + * will go wrong! + * + * if (fs != null) { + * try { + * fs.close(); + * } catch (IOException e) { + * e.printStackTrace(); + * } + * } + */ + } + + LOG.info(sb.toString()); + + return new MonitorItemData>( + MONITOR_ITEM_NAME, resMap, uploadBegTimestamp); + + } + + @Override + public MonitorItemData> extract(MonitorItemData> rawData) { + return rawData; + } + + + + + + /** + * for unit testing + * + * @param args + */ + public static void main(String[] args) { + /*HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker worker = + new HDFS_RPC_FileSystem_FileUploadAndRemoveBenchmark_MonitorItemWorker(); + worker.extract(worker.collect());*/ + + } + +} + diff --git a/src/java/hma/monitor/collection/custom/HDFS_RtExec_Peta2_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker.java b/src/java/hma/monitor/collection/custom/HDFS_RtExec_Peta2_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker.java new file mode 100644 index 0000000..ced2f1d --- /dev/null +++ b/src/java/hma/monitor/collection/custom/HDFS_RtExec_Peta2_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker.java @@ -0,0 +1,291 @@ +package hma.monitor.collection.custom; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.http.client.ClientProtocolException; +import org.json.JSONException; +import org.json.JSONObject; + + + + +import hma.monitor.MonitorManager; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemWorker; +import hma.util.EasyWebGet; + +public class HDFS_RtExec_Peta2_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker +extends MonitorItemWorker>, List>> { + + public static final String MONITOR_ITEM_NAME = + "HDFS_RtExec_Peta2_RecentlyRsyncedMetadataFileStatus"; + + + Map nnFmsMetaDataMap = new HashMap (); + public static class ProcKiller extends Thread{ + private Process proc; + + public ProcKiller(Process proc) + { + this.proc = proc; + } + + public void run() { + + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + proc.destroy(); + } + } + + class RtCollector extends Thread{ + private String commandArgs = null; + + public RtCollector(String commandArgs) + { + this.commandArgs = commandArgs; + + } + private synchronized void doCollect(){ + StringBuilder sb1 = new StringBuilder(); + for(int retry = 1; retry < 4; retry++ ){ + String host = commandArgs.split(":")[0]; + String metaName = commandArgs.split(":")[1]; + Runtime rt = null; + rt = Runtime.getRuntime(); + + InputStream in = null; + BufferedReader reader = null; + + String [] command = {"lftp", host,"-e","cd /home/work/hadoop-peta2-rsync/meta && ls -t " + metaName +".[0-9]*[0-9] && quit"}; + + + int count = 0; + try { + + Process proc = rt.exec(command); + //System.out.println("exitValue is : " + proc.exitValue()); + //ExecutorService exec = Executors.newCachedThreadPool(); + //exec.execute(ProcKiller(proc)); + Thread thread = new ProcKiller(proc); + thread.start(); + /* + try{ + proc.waitFor(); + } catch (InterruptedException e) { + e.printStackTrace(); + }*/ + in = proc.getInputStream(); + + reader = new BufferedReader(new InputStreamReader(in)); + + String str = null; + count = 0; + while ((str = reader.readLine()) != null) { + count++; + sb1.append(str + "\n"); + if(count>0) + break; + + } + if(count ==0){ + System.out.println(" Peta2 collect rsync info & checkpoint log error !+++++error+++++error+++++ and retry times is : " + retry ); + } + + + + + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + }finally { + try { + if (reader != null) reader.close(); + if (in != null) in.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + if(count > 0 )break; + try { + RtCollector.sleep(10000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + + + } + + nnFmsMetaDataMap.put(commandArgs, sb1.toString()); + //System.out.println("kain's log about checkpoint stat " + host + ": \n" + sb1.toString()); + + } + + + public void run() { + + doCollect(); + } + } + + public static String getHostName(String ipString){ + + + InetAddress ipAddr = null; + + try { + + ipAddr = InetAddress.getByName(ipString); + } catch (UnknownHostException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return ipAddr.getHostName().toString(); + } + + + @Override + public MonitorItemData>> collect() { + long acquisitionTimestamp = System.currentTimeMillis(); + + + Map nnFmsMap = new HashMap(); + String adapterAddr = MonitorManager.getGlobalConf().get("dfs.peta2.adapter","nj01-nanling-hdfs.dmop.baidu.com"); + String zkUrl = "http://" + adapterAddr + ":8070/zkinfo"; + String zkRes = null; + JSONObject additionalInfo = new JSONObject(); + StringBuilder hostlistBuilder = new StringBuilder(); + try { + zkRes = EasyWebGet.get(zkUrl); + } catch (ClientProtocolException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + JSONObject zkJson = null; + try { + zkJson = new JSONObject(zkRes); + } catch (JSONException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + Iterator iter = zkJson.keys(); + StringBuilder fmsSb = new StringBuilder(); + while (iter.hasNext()) { + String zkKey = (String) iter.next(); + //System.out.println(zkKey); + if (zkKey.contains("ACTIVE") ) { + String avIp = null; + String sbIp = null; + try { + if ( (((JSONObject) zkJson.get(zkKey)).get("lastNotNull")).equals(null) ){ + continue; + } + avIp = ((String) ((JSONObject) zkJson.get(zkKey)).get("lastNotNull")).replaceAll(":\\d+",""); + sbIp = ((String) ((JSONObject) zkJson.get(zkKey.replace("ACTIVE", "STANDBY"))).get("lastNotNull")).replaceAll(":\\d+",""); + String avHost = getHostName(avIp).replace(".baidu.com", ""); + String sbHost = getHostName(sbIp).replace(".baidu.com", ""); + nnFmsMap.put(avHost, sbHost); + if (hostlistBuilder.length() > 0) hostlistBuilder.append(","); + hostlistBuilder.append(avHost+","+sbHost); + //System.out.println(avHost + " " + sbHost); + } catch (JSONException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + } + + //collect_meta_data cmd = new collect_meta_data(); + + String metaNames [] = {"edits","fsimage"}; + List tList = new ArrayList(); + Iterator> nnFmsiter = nnFmsMap.entrySet().iterator(); + while(nnFmsiter.hasNext()){ + Map.Entry entry = nnFmsiter.next(); + + String[] hosts = {entry.getKey(),entry.getValue()}; + for (String host:hosts){ + for(String metaName:metaNames){ + //Thread rc = new RtCollector(host+":"+metaName); + Thread rc = new RtCollector(host+ ":" + metaName); + rc.start(); + tList.add(rc); + } + } + + + } + for(int i=0;i> nnFmsiter1 = nnFmsMap.entrySet().iterator(); + while(nnFmsiter1.hasNext()){ + Map.Entry entry = nnFmsiter1.next(); + for (String metaName:metaNames){ + String avMM [] = nnFmsMetaDataMap.get(entry.getKey() + ":" + metaName).split("\\s+"); + String sbMM [] = nnFmsMetaDataMap.get(entry.getValue() + ":" + metaName).split("\\s+"); + System.out.println(entry.getKey() + + " " + + avMM[avMM.length -4 ] + " " + avMM[avMM.length -3 ] + " " + + avMM[avMM.length -2 ] + " " + avMM[avMM.length -1 ] + + " " + + entry.getValue() + + " " + + sbMM[sbMM.length -4 ] + " " + sbMM[sbMM.length -3 ] + " " + + sbMM[sbMM.length -2 ] + " " + sbMM[sbMM.length -1 ] + ); + } + } */ + + List> resMapList = new ArrayList>(); + resMapList.add(nnFmsMap); + resMapList.add(nnFmsMetaDataMap); + try { + additionalInfo.put("additionalInfo", hostlistBuilder.toString()); + } catch (JSONException e) { + e.getCause(); + } + + return new MonitorItemData>>( + MONITOR_ITEM_NAME, resMapList, acquisitionTimestamp, additionalInfo.toString()); + // TODO Auto-generated method stub + + } + + + @Override + public MonitorItemData>> extract( + MonitorItemData>> rawData) { + // TODO Auto-generated method stub + return rawData; + } + + + + +} diff --git a/src/java/hma/monitor/collection/custom/HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker.java b/src/java/hma/monitor/collection/custom/HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker.java new file mode 100644 index 0000000..42b515f --- /dev/null +++ b/src/java/hma/monitor/collection/custom/HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker.java @@ -0,0 +1,214 @@ +/** + * + */ +package hma.monitor.collection.custom; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemWorker; +import hma.monitor.collection.metric.CollectionMetricPool; +import hma.monitor.collection.metric.CollectionMetricRecord; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + +/** + * @author guoyezhi + * + */ +public class HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker +extends MonitorItemWorker> { + + public static final String className = + HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker.class.getSimpleName(); + + public static final Log LOG = + LogFactory.getLog(HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker.class); + + public static final String MONITOR_ITEM_NAME = + "HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus"; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private Runtime rt = null; + + + public HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker() { + rt = Runtime.getRuntime(); + } + + + @Override + public MonitorItemData collect() { + + Configuration conf = MonitorManager.getGlobalConf(); + //String server = conf.get("rsynced.metadata.backup.machine"); + //String server = conf.getStrings("rsynced.metadata.backup.machine"); + //String[] server = conf.get("rsynced.metadata.backup.machine").split(","); + String server = null; + String[] machine_dir = conf.get("rsynced.metadata.backup.machine.and.dir").split(","); + + //if (server == null) { throw new RuntimeException(); } + +// String dir = conf.get( "recently.rsynced.metadata.directory","/home/work/hadoop-rsync/meta/name/current"); + + StringBuffer buffer = new StringBuffer(); + StringBuilder sb = new StringBuilder(); + long acquisitionTimestamp = System.currentTimeMillis(); + long begTimestamp = acquisitionTimestamp; + long endTimestamp = 0; + + InputStream in = null; + //InputStream err = null; + + BufferedReader reader = null; + for(int i = 0; i < machine_dir.length; i++) { + String[] one_machine_dir = machine_dir[i].split(":"); + server = one_machine_dir[0]; + int count =0; + for(int j = 1; j < one_machine_dir.length; j++) { + String dir = one_machine_dir[j]; + try { + + sb.append("\n\t[" + dateFormat.format(new Date(begTimestamp)) + " - " + + className + "] Getting " + MONITOR_ITEM_NAME + " ..."); + in = rt.exec("safewrapper 12 /usr/bin/ssh " + + "-o StrictHostKeyChecking=no " + + "-o PasswordAuthentication=no " + + "-o ConnectTimeout=10 " + + "rd@" + + server + " " + + "stat -t " + dir + "/*").getInputStream(); + reader = new BufferedReader(new InputStreamReader(in)); + String str = null; + count = 0; + while ((str = reader.readLine()) != null) { + count++; + sb.append("\n\t[" + dateFormat.format(new Date(begTimestamp)) + " - " + + className + "] -> server: "+ server + " " + str + "\n"); + buffer.append(str + " " + server + ":" + dir + "\n"); + //buffer.append(str + "\n"); + } + if(count ==0){ + LOG.fatal(MONITOR_ITEM_NAME+" this "+server+" collect error !+++++error+++++error+++++"); + return null; + } + endTimestamp = System.currentTimeMillis(); + sb.append("\n\t[" + dateFormat.format(new Date(endTimestamp)) + " - " + + className + "] Have got " + MONITOR_ITEM_NAME); + + } catch (Throwable e) { + e.printStackTrace(); + return null; + } finally { + try { + if (reader != null) reader.close(); + if (in != null) in.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + CollectionMetricPool.offerRecord( + new CollectionMetricRecord( + MonitorManager.getMonitoredClusterName(), + MONITOR_ITEM_NAME, + begTimestamp, + endTimestamp - begTimestamp)); + + LOG.info(sb.toString()); + LOG.info(buffer.toString()); + + if (buffer.length() > 0) { + return new MonitorItemData( + MONITOR_ITEM_NAME, buffer.toString(), acquisitionTimestamp); + } + + return null; + } + + @Override + public MonitorItemData> extract(MonitorItemData rawData) { + + Configuration conf = MonitorManager.getGlobalConf(); + String dir = conf.get( + "recently.rsynced.metadata.directory", + "/home/work/hadoop-rsync/meta/name/current"); + + String[] fsArray = rawData.getData().split("\n"); + List fsList = new ArrayList(); + + for (int i = 0; i < fsArray.length; i++) { + + /* + * Size:4935289049 Access:1281627657 Modify:1290958135 + * file name | size | blocks | |uid|gid|dev| inode | | | | access | modify | change | + * /home/work/hadoop-rsync/meta/last_minutes/fsimage 13348683493 26097152 81b4 503 505 803 35438610 1 0 0 1292418968 1292803033 1292803033 4096 + */ + String[] statusArray = fsArray[i].split(" "); + + if (statusArray.length == 16) { +// if (statusArray.length == 15 && statusArray[0].startsWith(dir)) { + String fname = statusArray[0]; + long size = Long.parseLong(statusArray[1]); + long modificationTime = Long.parseLong(statusArray[12]) * 1000; + String owner = statusArray[15]; + + //fsList.add(new FileStatus(size, false, 0, 0, modificationTime, new Path(fname))); + fsList.add(new FileStatus(size, false, 0, 0, modificationTime, 0, null, owner, null, new Path(fname))); + } else { + LOG.warn("Invalid RawData Format: <" + + fsArray[i] + + "> : Total " + statusArray.length + " fields"); + } + + } + + if (fsList.size() > 0) { + return new MonitorItemData>(rawData.getName(), + fsList, rawData.getTimestamp()); + } + + LOG.error("Extract Data Failed !"); + + return null; + } + + + /** + * for unit testing + * + * @param args + * @throws ParseException + */ + public static void main(String[] args) throws ParseException { + + HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker worker = + new HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus_MonitorItemWorker(); + + FileStatus status = + (FileStatus) worker.extract(worker.collect()).getData(); + System.out.println("Fsimage Size: " + status.getLen()); + System.out.println("Fsimage modified at: " + + new Date(status.getModificationTime())); + + } + +} + diff --git a/src/java/hma/monitor/collection/custom/MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker.java b/src/java/hma/monitor/collection/custom/MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker.java new file mode 100644 index 0000000..cfaa101 --- /dev/null +++ b/src/java/hma/monitor/collection/custom/MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker.java @@ -0,0 +1,299 @@ +package hma.monitor.collection.custom; + +import hma.monitor.MonitorManager; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.collection.MonitorItemWorker; +import hma.monitor.collection.metric.CollectionMetricPool; +import hma.monitor.collection.metric.CollectionMetricRecord; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; + + + + +import org.apache.hadoop.mapreduce.*; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; + +import static org.apache.hadoop.mapreduce.JobPriority.VERY_HIGH; + + +/** + * @author kain + * + */ +public class MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker extends MonitorItemWorker< + MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker.HMAProbingJobResult, + MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker.HMAProbingJobResult> { + + public static final String className = + MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker.class.getSimpleName(); + + public static final Log LOG = + LogFactory.getLog(MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker.class); + + public static final String MONITOR_ITEM_NAME = + "MapRed_RPC_ComputingStatus_AvailabilityProbingJob"; + + private Configuration hadoopConf = null; + + private static SimpleDateFormat jobDateFormat = + new SimpleDateFormat("yyyyMMddHHmmssSSS"); + + + public MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker() { + String hadoopConfDir = + MonitorManager.getMonitorManager().getConf().get( + "hadoop.client.conf.dir"); + + hadoopConf = new Configuration(false); + hadoopConf.addResource(new Path(hadoopConfDir, "core-site.xml")); + hadoopConf.addResource(new Path(hadoopConfDir, "hdfs-site.xml")); + hadoopConf.addResource(new Path(hadoopConfDir, "mapred-site.xml")); + hadoopConf.addResource(new Path(hadoopConfDir, "yarn-site.xml")); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-default.xml")); + hadoopConf.addResource(new Path(hadoopConfDir, "hadoop-site.xml")); + } + + + + + + + + public static class HMAProbingJobResult { + + private JobID jobID = null; + + private String jobName = null; + + private String queueName = null; + + private boolean successful = false; + + private Date jobSubmitTime = null; + + private Date jobCompleteTime = null; + + public HMAProbingJobResult( + JobID jobID, + String jobName, + String queueName, + boolean successful, + Date jobSubmitTime, + Date jobCompleteTime) { + this.jobID = jobID; + this.jobName = jobName; + this.queueName = queueName; + this.successful = successful; + this.jobSubmitTime = jobSubmitTime; + this.jobCompleteTime = jobCompleteTime; + } + + public JobID getJobID() { + return jobID; + } + + public String getJobName() { + return jobName; + } + + public String getQueueName() { + return queueName; + } + + public boolean isSuccessful() { + return successful; + } + + public Date getJobSubmitTime() { + return jobSubmitTime; + } + + public Date getJobCompleteTime() { + return jobCompleteTime; + } + + } + + + /* + * (non-Javadoc) + * + * @see hma.monitor.collection.MonitorItemWorker#collect() + */ + @Override + public MonitorItemData collect() { + + boolean jobSuccessful = false; + hma.conf.Configuration hmaConf = MonitorManager.getGlobalConf(); + Date submitTime = new Date(); + Date completeTime = null; + boolean jobError = false; + SimpleDateFormat jobDateFormat = + new SimpleDateFormat("yyyyMMddHHmmssSSS"); + String jobTimestampID = jobDateFormat.format(System.currentTimeMillis()); + String hadoopConfDir = + MonitorManager.getMonitorManager().getConf().get( + "hadoop.client.conf.dir"); + Configuration conf = new Configuration(); + conf.addResource(new Path(hadoopConfDir, "core-site.xml")); + conf.addResource(new Path(hadoopConfDir, "hdfs-site.xml")); + conf.addResource(new Path(hadoopConfDir, "mapred-site.xml")); + conf.addResource(new Path(hadoopConfDir, "yarn-site.xml")); + conf.addResource(new Path(hadoopConfDir, "hadoop-default.xml")); + conf.addResource(new Path(hadoopConfDir, "hadoop-site.xml")); + + Job job = null; + JobID jobID = null; + String jobName = null; + String queueName = ""; + + try { + job = Job.getInstance(conf, "HMA_ProbingJob_" + jobTimestampID); + job.setInputFormatClass(org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat.class); + job.setOutputFormatClass(org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.class); + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(Text.class); + job.setMapperClass(org.apache.hadoop.mapreduce.Mapper.class); + job.setReducerClass(org.apache.hadoop.mapreduce.Reducer.class); + job.setNumReduceTasks(1); + job.setPriority(VERY_HIGH); + //job.setJar(hmaConf.get("probing.job.jar.path", "../../lib/hma.jar")); + org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(job, new Path("/monitor/test")); + org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.setOutputPath(job, new Path("/monitor/out." + jobTimestampID)); + try { + job.submit(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + jobID = job.getJobID(); + jobName = job.getJobName(); + + System.out.println("job url: " + job.getTrackingURL() + " sch info: " + + job.getSchedulingInfo() + " job history: " + + job.getHistoryUrl() + " job name: " + job.getJobName() + + " job id: " + jobID); + int tryTimes = 0; + int maxTryTimes = conf.getInt("stream.report.maxretry", 300); + + while (true) { + try { + + if (job.isComplete()) { + completeTime = new Date(); + break; + } + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + // nothing needed to do + } + + LOG.info("map " + Math.round(job.mapProgress() * 100) + "%\t" + + "reduce " + Math.round(job.reduceProgress() * 100) + "%"); + tryTimes = 0; + + + } catch (IOException ioe) { + if (tryTimes++ >= maxTryTimes) { + // not an error + jobError = false; + break; + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + } + System.out.println("job is complete " + job.isComplete() + + " job is successful " + job.isSuccessful()); + + + + + if (!job.isSuccessful()) { + jobSuccessful = false; + hma.util.LOG.warn("Job not Successful: " + jobID + " JobQueue: " ); + } else { + jobSuccessful = true; + hma.util.LOG.info("Job Complete Successfully: " + jobID + " JobQueue: " ); + } + + jobError = false; + + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + }finally{ + if (jobError && (job != null)) { + try { + LOG.error("There's something wrong. Kill job: " + jobID + " JobQueue " ); + job.killJob(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + } + + + + CollectionMetricPool.offerRecord( + new CollectionMetricRecord( + MonitorManager.getMonitoredClusterName(), + MONITOR_ITEM_NAME, + submitTime.getTime(), + completeTime.getTime() - submitTime.getTime())); + + HMAProbingJobResult jobResult = + new HMAProbingJobResult( + jobID, jobName, queueName, + jobSuccessful, submitTime, completeTime); + + return new MonitorItemData( + MONITOR_ITEM_NAME, jobResult, submitTime.getTime()); + + } + + /* + * (non-Javadoc) + * + * @see + * hma.monitor.collection.MonitorItemWorker#extract(hma.monitor.collection + * .MonitorItemData) + */ + @Override + public MonitorItemData extract( + MonitorItemData rawData) { + return rawData; + } + + + /** + * for unit testing + * + * @param args + */ + public static void main(String[] args) { + MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker worker = + new MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker(); + HMAProbingJobResult jr = + worker.extract(worker.collect()).getData(); + System.out.println(jr.getJobID() + " " + jr.getJobName() + " " + + jr.getQueueName() + " Successful:" + jr.isSuccessful() + " " + + jr.getJobSubmitTime() + " " + jr.getJobCompleteTime()); + } + +} + diff --git a/src/java/hma/monitor/collection/custom/mr2.java b/src/java/hma/monitor/collection/custom/mr2.java new file mode 100644 index 0000000..1ea234a --- /dev/null +++ b/src/java/hma/monitor/collection/custom/mr2.java @@ -0,0 +1,141 @@ +package hma.monitor.collection.custom; + +import hma.monitor.MonitorManager; + +import hma.util.LOG; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * Created by weizhonghui on 2016/11/27. + */ +public class mr2 { + public void submit(){ + boolean jobSuccessful = false; + hma.conf.Configuration hmaConf = MonitorManager.getGlobalConf(); + Date submitTime = new Date(); + Date completeTime = null; + boolean jobError = false; + SimpleDateFormat jobDateFormat = + new SimpleDateFormat("yyyyMMddHHmmssSSS"); + String jobTimestampID = jobDateFormat.format(System.currentTimeMillis()); + String hadoopConfDir = "conf/CLUSTER-01"; + //MonitorManager.getMonitorManager().getConf().get( + // "hadoop.client.conf.dir"); + Configuration conf = new Configuration(); + conf.addResource(new Path(hadoopConfDir, "core-site.xml")); + conf.addResource(new Path(hadoopConfDir, "hdfs-site.xml")); + conf.addResource(new Path(hadoopConfDir, "mapred-site.xml")); + conf.addResource(new Path(hadoopConfDir, "yarn-site.xml")); + conf.addResource(new Path(hadoopConfDir, "hadoop-default.xml")); + conf.addResource(new Path(hadoopConfDir, "hadoop-site.xml")); + //System.out.println("check conf " + conf.get("fs.defaultFS")); + Job job = null; + try { + job = Job.getInstance(conf, "HmaTest"); + job.setInputFormatClass(KeyValueTextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(Text.class); + job.setMapperClass(Mapper.class); + job.setReducerClass(Reducer.class); + job.setNumReduceTasks(1); + //job.setJar(hmaConf.get("probing.job.jar.path", "../../lib/hma.jar")); + FileInputFormat.addInputPath(job, new Path("/monitor/test")); + FileOutputFormat.setOutputPath(job, new Path("/monitor/out." + jobTimestampID)); + try { + job.submit(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + String jobID = job.getJobID().toString(); + + System.out.println("job url: " + job.getTrackingURL() + " sch info: " + + job.getSchedulingInfo() + " job history: " + + job.getHistoryUrl() + " job name: " + job.getJobName() + + " job id: " + jobID); + int tryTimes = 0; + int maxTryTimes = conf.getInt("stream.report.maxretry", 300); + + while (true) { + try { + + if (job.isComplete()) { + completeTime = new Date(); + break; + } + try { + Thread.sleep(5000); + } catch (InterruptedException e) { +// nothing needed to do + } + +//LOG.info("map " + Math.round(job.mapProgress() * 100) + "%\t" + +//"reduce " + Math.round(job.reduceProgress() * 100) + "%"); + tryTimes = 0; + + + } catch (IOException ioe) { + if (tryTimes++ >= maxTryTimes) { +// not an error + jobError = false; + break; + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + } + System.out.println("job is complete " + job.isComplete() + " job is successful " + job.isSuccessful()); + + + + if (!job.isSuccessful()) { + jobSuccessful = false; + LOG.warn("Job not Successful: " + jobID + " JobQueue: " ); + } else { + jobSuccessful = true; + LOG.info("Job Complete Successfully: " + jobID + " JobQueue: " ); + } + + jobError = false; + + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + }finally{ + if (jobError && (job != null)) { + try { + //LOG.error("There's something wrong. Kill job: " + jobID + " JobQueue: " + queueName); + job.killJob(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + } + + } + public static void main(String [] args ){ + mr2 m = new mr2(); + m.submit(); + + } +} + diff --git a/src/java/hma/monitor/collection/metric/CollectionMetricDBDumper.java b/src/java/hma/monitor/collection/metric/CollectionMetricDBDumper.java new file mode 100644 index 0000000..3c8f55e --- /dev/null +++ b/src/java/hma/monitor/collection/metric/CollectionMetricDBDumper.java @@ -0,0 +1,109 @@ +/** + * + */ +package hma.monitor.collection.metric; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.TimeUnit; + +/** + * @author guoyezhi + * + */ +public class CollectionMetricDBDumper implements Runnable { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd" + " " + "HH:mm:ss"); + + private String dbServer = null; + private String dbName = null; + private String tableName = null; + private String userName = null; + private String password = null; + + /** + * + */ + public CollectionMetricDBDumper() { + } + + private Connection initConnection() throws SQLException { + + Configuration hmaConf = + MonitorManager.getMonitorManager().getConf(); + + dbServer = hmaConf.get( + "monitor.data.db.server", "yf-2950-hma-db.yf01.baidu.com"); + dbName = hmaConf.get( + "monitor.data.db.name", "hma"); + tableName = hmaConf.get( + "monitor.collection.metric.db.table", + "monitor_cmetric"); + userName = hmaConf.get( + "monitor.db.user.name", "root"); + password = hmaConf.get( + "monitor.db.user.pass", "hmainit"); + + return DriverManager.getConnection( + "jdbc:mysql://" + dbServer + "/" + dbName, + userName, + password); + } + + @Override + public void run() { + + Connection dbConn = null; + try { + dbConn = this.initConnection(); + } catch (SQLException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + while (true) { + CollectionMetricRecord record = + CollectionMetricPool.pollRecord(); + if (record == null) { + try { + TimeUnit.SECONDS.sleep(60); + } catch (InterruptedException e) { + e.printStackTrace(); + } + continue; + } + PreparedStatement pstat = null; + try { + String detectionTimestamp = + dateFormat.format(new Date(record.getDetectionTimestamp())); + pstat = dbConn.prepareStatement( + "INSERT INTO " + tableName + " VALUES ( ?, ?, ?, ? )"); + pstat.setString(1, record.getClusterName()); + pstat.setString(2, record.getItemName()); + pstat.setString(3, detectionTimestamp); + pstat.setLong(4, record.getTimeConsumed()); + pstat.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } finally { + if (pstat != null) { + try { + pstat.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + } + + } + +} diff --git a/src/java/hma/monitor/collection/metric/CollectionMetricPool.java b/src/java/hma/monitor/collection/metric/CollectionMetricPool.java new file mode 100644 index 0000000..9143b5d --- /dev/null +++ b/src/java/hma/monitor/collection/metric/CollectionMetricPool.java @@ -0,0 +1,43 @@ +/** + * + */ +package hma.monitor.collection.metric; + +import hma.monitor.MonitorManager; + +import java.util.LinkedList; +import java.util.Queue; + +/** + * @author guoyezhi + * + */ +public class CollectionMetricPool { + + private static Queue pool = + new LinkedList(); + + /* + * `MonitorManager' class is always loaded before `CollectionMetricPool' + * class. + */ + private static boolean metricCollected = + MonitorManager.getGlobalConf().getBoolean( + "monitor.collection.metric.enable", true); + + public synchronized static void offerRecord(CollectionMetricRecord record) { + if (metricCollected) { + // just discard record if not wanna collect + pool.offer(record); + } + } + + public synchronized static CollectionMetricRecord pollRecord() { + if (metricCollected) { + return pool.poll(); + } else { + return null; + } + } + +} diff --git a/src/java/hma/monitor/collection/metric/CollectionMetricRecord.java b/src/java/hma/monitor/collection/metric/CollectionMetricRecord.java new file mode 100644 index 0000000..a7b8efc --- /dev/null +++ b/src/java/hma/monitor/collection/metric/CollectionMetricRecord.java @@ -0,0 +1,47 @@ +/** + * + */ +package hma.monitor.collection.metric; + +/** + * @author guoyezhi + * + */ +public class CollectionMetricRecord { + + private String clusterName = null; + + private String itemName = null; + + private long detectionTimestamp = 0; + + private long timeConsumed = -1; + + public CollectionMetricRecord( + String clusterName, + String itemName, + long detectionTimestamp, + long timeConsumed) { + this.clusterName = clusterName; + this.itemName = itemName; + this.detectionTimestamp = detectionTimestamp; + this.timeConsumed = timeConsumed; + } + + public String getClusterName() { + return clusterName; + } + + public String getItemName() { + return itemName; + } + + public long getDetectionTimestamp() { + return detectionTimestamp; + } + + public long getTimeConsumed() { + return timeConsumed; + } + +} diff --git a/src/java/hma/monitor/strategy/MonitorStrategyConfiguration.java b/src/java/hma/monitor/strategy/MonitorStrategyConfiguration.java new file mode 100644 index 0000000..80f5a83 --- /dev/null +++ b/src/java/hma/monitor/strategy/MonitorStrategyConfiguration.java @@ -0,0 +1,329 @@ +/** + * + */ +package hma.monitor.strategy; + +import hma.conf.Configuration; +import hma.monitor.strategy.MonitorStrategyPrototype.AnomalyDetectionStrategy; +import hma.monitor.strategy.MonitorStrategyPrototype.DataAcquisitionStrategy; +import hma.monitor.strategy.MonitorStrategyPrototype.TaskTriggerStrategy; +import hma.util.LOG; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; + +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; + +/** + * @author guoyezhi + * + */ +public class MonitorStrategyConfiguration extends Configuration { + + private HashMap prototypes = null; + + public MonitorStrategyConfiguration() { + this(true); + } + + /* + public MonitorStrategyConfiguration(boolean loadDefaults) { + super(false); + List resources = this.getResources(); + if (loadDefaults) { + resources.add("D:\\eclipse\\guoyezhi\\workspace\\HMA\\conf\\monitor-strategies-default.xml"); + //resources.add("D:\\eclipse\\guoyezhi\\workspace\\HMA\\src\\monitor-items-site.xml"); + this.setResources(resources); + } + // LOG.debug(resources.toString()); + } + */ + + /* + * TODO: + */ + public MonitorStrategyConfiguration(boolean loadDefaults) { + super(loadDefaults, "monitor-strategies"); + } + + + public synchronized HashMap getAllPrototypes() { + if (prototypes == null) { + prototypes = getPrototypes(); + } + return new HashMap(prototypes); // return a copy + } + + private synchronized HashMap getPrototypes() { + + if (prototypes == null) { + + prototypes = new HashMap(); + + List resources = this.getResources(); + boolean quietmode = this.getQuietMode(); + List list = loadResources(resources, quietmode); + + Iterator iter = list.iterator(); + while(iter.hasNext()) { + + Element elem = iter.next(); + if (!"monitor_strategy".equals(elem.getTagName())) { + LOG.warn("bad monitor strategy conf file: element not "); + continue; + } + + this.generatePrototype(elem, prototypes); + } + } + + return prototypes; + } + + + private static final String DEFAULT_MONITOR_PERIOD_CONF = "60000"; + + private static final String DEFAULT_ALARM_LEVEL_CONF = "WARNING"; + + /** + * Internal core implementation of configuration parsing + * & prototype generation + */ + private synchronized void generatePrototype(Element entryConf, + HashMap prototypes) { + + NodeList propNodes = entryConf.getChildNodes(); + + String nameConf = null; + String aliasConf = null; + String descriptionConf = null; + String periodConf = DEFAULT_MONITOR_PERIOD_CONF; // monitor_period + String levelConf = DEFAULT_ALARM_LEVEL_CONF; // alarm_level + MonitorStrategyPrototype.DataAcquisitionStrategy daStrategy = null; + MonitorStrategyPrototype.AnomalyDetectionStrategy adStrategy = null; + MonitorStrategyPrototype.TaskTriggerStrategy ttStrategy = null; + + Element daConf = null; + Element adConf = null; + Element ttConf = null; + + for (int i = 0; i < propNodes.getLength(); i++) { + + Node propNode = propNodes.item(i); + if (!(propNode instanceof Element)) + continue; + + Element prop = (Element)propNode; + if ("strategy_name".equals(prop.getTagName()) && prop.hasChildNodes()) + nameConf = ((Text)prop.getFirstChild()).getData().trim(); + if ("strategy_alias".equals(prop.getTagName()) && prop.hasChildNodes()) + aliasConf = ((Text)prop.getFirstChild()).getData().trim(); + if ("strategy_description".equals(prop.getTagName()) && prop.hasChildNodes()) + descriptionConf = ((Text)prop.getFirstChild()).getData().trim(); + if ("monitor_period".equals(prop.getTagName()) && prop.hasChildNodes()) + periodConf = ((Text)prop.getFirstChild()).getData().trim(); + if ("alarm_level".equals(prop.getTagName()) && prop.hasChildNodes()) + levelConf = ((Text)prop.getFirstChild()).getData().trim(); + if ("data_acquisition".equals(prop.getTagName()) && prop.hasChildNodes()) + daConf = prop; + if ("anomaly_detection".equals(prop.getTagName()) && prop.hasChildNodes()) + adConf = prop; + if ("task_trigger".equals(prop.getTagName()) && prop.hasChildNodes()) + ttConf = prop; + } + + if (nameConf == null) { + if (this.getQuietMode() == false) { + LOG.warn("Syntax errors in entry conf: No strategy name configured."); + } + return; + } + + MonitorStrategyPrototype msp = new MonitorStrategyPrototype( + nameConf, aliasConf, descriptionConf, periodConf, levelConf); + + daStrategy = this.parseDataAcquisitionConf(daConf, msp); + adStrategy = this.parseAnomalyDetectionConf(adConf, msp); + ttStrategy = this.parseTaskTriggerConf(ttConf, msp); + + if (daStrategy != null && adStrategy != null && ttStrategy != null) { + msp.setDataAcquisitionStrategy(daStrategy); + msp.setAnomalyDetectionStrategy(adStrategy); + msp.setTaskTriggerStrategy(ttStrategy); + prototypes.put(nameConf, msp); + return; + } + + if (this.getQuietMode() == false) { + LOG.warn("Syntax errors in entry conf: " + nameConf); + LOG.warn("daStrategy = " + daStrategy); + LOG.warn("adStrategy = " + adStrategy); + LOG.warn("ttStrategy = " + ttStrategy); + } + } + + private DataAcquisitionStrategy parseDataAcquisitionConf( + Element e, MonitorStrategyPrototype prototype) { + + LOG.warn("e = " + e); + + DataAcquisitionStrategy daStrategy = + prototype.new DataAcquisitionStrategy(); + + NodeList itemNodes = e.getChildNodes(); + for (int i = 0; i < itemNodes.getLength(); i++) { + + Node itemNode = itemNodes.item(i); + if (!(itemNode instanceof Element)) + continue; + Element item = (Element) itemNode; + if ("item".equals(item.getTagName()) == false) { + continue; + } + + NodeList fieldNodes = item.getChildNodes(); + String itemName = null; + String timeout = null; + List attachedTaskConfs = new ArrayList(); + for (int j = 0; j < fieldNodes.getLength(); j++) { + Node fieldNode = fieldNodes.item(j); + if (!(fieldNode instanceof Element)) + continue; + Element field = (Element)fieldNode; + if ("name".equals(field.getTagName()) && field.hasChildNodes()) + itemName = ((Text)field.getFirstChild()).getData().trim(); + if ("timeout".equals(field.getTagName()) && field.hasChildNodes()) + timeout = ((Text)field.getFirstChild()).getData().trim(); + if ("attached_tasks".equals(field.getTagName()) && field.hasChildNodes()) { + NodeList subFieldNodes = field.getChildNodes(); + for (int k = 0; k < subFieldNodes.getLength(); k++) { + Node subFieldNode = subFieldNodes.item(k); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("task".equals(subField.getTagName()) + && subField.hasChildNodes()) + attachedTaskConfs.add(subField); + } + } + } + + if (itemName != null && timeout != null) { + daStrategy.putMeta(itemName, timeout, attachedTaskConfs); + } + + LOG.warn("itemName = " + itemName); + LOG.warn("timeout = " + timeout); + } + + if (daStrategy.getAllInvolvedItemNumber() > 0) { + return daStrategy; + } + + return null; + } + + private AnomalyDetectionStrategy parseAnomalyDetectionConf( + Element e, MonitorStrategyPrototype prototype) { + + LOG.warn("e = " + e); + + AnomalyDetectionStrategy adStrategy = + prototype.new AnomalyDetectionStrategy(); + + NodeList itemNodes = e.getChildNodes(); + for (int i = 0; i < itemNodes.getLength(); i++) { + + Node itemNode = itemNodes.item(i); + if (!(itemNode instanceof Element)) + continue; + Element item = (Element)itemNode; + if ("item".equals(item.getTagName()) == false) { + continue; + } + + NodeList fieldNodes = item.getChildNodes(); + + String monitorItemName = null; + String monitorDataType = null; + String monitorDataSubType = null; + Element strategy = null; + + for (int j = 0; j < fieldNodes.getLength(); j++) { + Node fieldNode = fieldNodes.item(j); + if (!(fieldNode instanceof Element)) + continue; + Element field = (Element)fieldNode; + if ("name".equals(field.getTagName()) && field.hasChildNodes()) + monitorItemName = ((Text)field.getFirstChild()).getData().trim(); + if ("type".equals(field.getTagName()) && field.hasChildNodes()) + monitorDataType = ((Text)field.getFirstChild()).getData().trim(); + if ("sub-type".equals(field.getTagName()) && field.hasChildNodes()) + monitorDataSubType = ((Text)field.getFirstChild()).getData().trim(); + if ("strategy".equals(field.getTagName()) && field.hasChildNodes()) + strategy = field; + } + + if (monitorItemName != null && monitorDataType != null + && strategy != null) { + adStrategy.putMeta(monitorItemName, + monitorDataType, monitorDataSubType, strategy); + } + + //LOG.warn("monitorItemName = " + monitorItemName); + //LOG.warn("monitorDataType = " + monitorDataType); + //LOG.warn("monitorDataSubType = " + monitorDataSubType); + } + + if (adStrategy.getAllInvolvedAnomalyDetectors().size() > 0) { + return adStrategy; + } + + return null; + } + + private TaskTriggerStrategy parseTaskTriggerConf(Element e, + MonitorStrategyPrototype prototype) { + + NodeList fieldNodes = e.getChildNodes(); + + String mode = null; + Element strategy = null; + List attachedTaskConfs = new ArrayList(); + + for (int i = 0; i < fieldNodes.getLength(); i++) { + Node fieldNode = fieldNodes.item(i); + if (!(fieldNode instanceof Element)) + continue; + Element field = (Element)fieldNode; + if ("mode".equals(field.getTagName()) && field.hasChildNodes()) + mode = ((Text)field.getFirstChild()).getData().trim(); + if ("strategy".equals(field.getTagName()) && field.hasChildNodes()) + strategy = field; + if ("attached_tasks".equals(field.getTagName()) && field.hasChildNodes()) { + NodeList subFieldNodes = field.getChildNodes(); + for (int k = 0; k < subFieldNodes.getLength(); k++) { + Node subFieldNode = subFieldNodes.item(k); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("task".equals(subField.getTagName()) + && subField.hasChildNodes()) + attachedTaskConfs.add(subField); + } + } + } + + if (mode != null && (attachedTaskConfs.size() > 0)) { + return prototype.new TaskTriggerStrategy(mode, strategy, attachedTaskConfs); + } + + return null; + } + +} diff --git a/src/java/hma/monitor/strategy/MonitorStrategyData.java b/src/java/hma/monitor/strategy/MonitorStrategyData.java new file mode 100644 index 0000000..262e179 --- /dev/null +++ b/src/java/hma/monitor/strategy/MonitorStrategyData.java @@ -0,0 +1,99 @@ +/** + * + */ +package hma.monitor.strategy; + +import hma.monitor.collection.MonitorItemData; + +/** + * @author guoyezhi + * add additionalInfo by chiwen01 + * + */ +public class MonitorStrategyData { + + private String itemName = null; + + private TYPE data = null; + + private long timestamp = 0; + + private String additionalInfo = null; + + public MonitorStrategyData(MonitorItemData itemData) { + this.itemName = itemData.getName(); + this.data = itemData.getData(); + this.timestamp = itemData.getTimestamp(); + this.additionalInfo = itemData.getAdditionalInfo(); + } + + public MonitorStrategyData(String itemName, TYPE data, long timestamp){ + this.itemName = itemName; + this.data = data; + this.timestamp = timestamp; + } + + public MonitorStrategyData(String itemName, TYPE data, long timestamp, String additionalInfo){ + this.itemName = itemName; + this.data = data; + this.timestamp = timestamp; + this.additionalInfo = additionalInfo; + } + + /** + * @param itemName the itemName to set + */ + public void setItemName(String itemName) { + this.itemName = itemName; + } + + /** + * @return the itemName + */ + public String getItemName() { + return itemName; + } + + /** + * @param data the data to set + */ + public void setData(TYPE data) { + this.data = data; + } + + /** + * @return the data + */ + public TYPE getData() { + return data; + } + + /** + * @param timestamp the timestamp to set + */ + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + /** + * @return the timestamp + */ + public long getTimestamp() { + return timestamp; + } + + /** + * @param additionalInfo the additionalInfo to set + */ + public void setAdditionalInfo(String additionalInfo) { + this.additionalInfo = additionalInfo; + } + + /** + * @return the additionalInfo + */ + public String getAdditionalInfo() { + return additionalInfo; + } + +} diff --git a/src/java/hma/monitor/strategy/MonitorStrategyDataCollection.java b/src/java/hma/monitor/strategy/MonitorStrategyDataCollection.java new file mode 100644 index 0000000..226c8f2 --- /dev/null +++ b/src/java/hma/monitor/strategy/MonitorStrategyDataCollection.java @@ -0,0 +1,70 @@ +/** + * + */ +package hma.monitor.strategy; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * support mapping M items to ONE strategy + * + * @author guoyezhi + * + */ +public class MonitorStrategyDataCollection { + + private String strategyName; + + private Map> collection = + new HashMap>(); + + private boolean anomalous = false; + + + public MonitorStrategyDataCollection(String strategyName) { + this.strategyName = strategyName; + } + + public Set getInvolvedMonitorItemNames() { + return this.collection.keySet(); + } + + public MonitorStrategyData put(String itemName, MonitorStrategyData data) { + return this.collection.put(itemName, data); + } + + public MonitorStrategyData get(String itemName) { + return this.collection.get(itemName); + } + + /** + * @param strategyName the strategyName to set + */ + public void setStrategyName(String strategyName) { + this.strategyName = strategyName; + } + + /** + * @return the strategyName + */ + public String getStrategyName() { + return strategyName; + } + + /** + * @param anomalous the anomalous to set + */ + public void setAnomalous(boolean anomalous) { + this.anomalous = anomalous; + } + + /** + * @return the anomalous + */ + public boolean isAnomalous() { + return anomalous; + } + +} diff --git a/src/java/hma/monitor/strategy/MonitorStrategyPrototype.java b/src/java/hma/monitor/strategy/MonitorStrategyPrototype.java new file mode 100644 index 0000000..4c744f3 --- /dev/null +++ b/src/java/hma/monitor/strategy/MonitorStrategyPrototype.java @@ -0,0 +1,512 @@ +/** + * + */ +package hma.monitor.strategy; + +import hma.monitor.strategy.detect.AnomalyDetector; +import hma.monitor.strategy.detect.AnomalyDetectorGenerator; +import hma.monitor.strategy.trigger.TaskTrigger; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapterGenerater; +import hma.util.LOG; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.w3c.dom.Element; + +/** + * @author guoyezhi + * + */ +public class MonitorStrategyPrototype { + + private String name = null; + + private String alias = null; + + private String description = null; + + private long monitorPeriod = 0; + + private int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_UNINIT; + + private DataAcquisitionStrategy daStrategy = null; + + private AnomalyDetectionStrategy adStrategy = null; + + private TaskTriggerStrategy ttStrategy = null; + + public MonitorStrategyPrototype( + String name, + String alais, + String description, + long monitorPeriod, + int alarmLevel) { + this.name = name; + this.alias = alais; + this.description = description; + this.monitorPeriod = monitorPeriod; + this.alarmLevel = alarmLevel; + } + + public MonitorStrategyPrototype( + String name, + String alais, + String description, + String monitorPeriod, + String alarmLevel) { + this(name, alais, description, Long.parseLong(monitorPeriod), + MonitorStrategyAttachedTaskAdapter.getAlarmLevelValueFromString(alarmLevel)); + } + + public MonitorStrategyPrototype( + String name, + String alais, + String description, + long monitorPeriod, + int alarmLevel, + DataAcquisitionStrategy daStrategy, + AnomalyDetectionStrategy adStrategy, + TaskTriggerStrategy ttStrategy) { + this(name, alais, description, monitorPeriod, alarmLevel); + this.setDataAcquisitionStrategy(daStrategy); + this.setAnomalyDetectionStrategy(adStrategy); + this.setTaskTriggerStrategy(ttStrategy); + } + + public MonitorStrategyPrototype( + String name, + String alais, + String description, + String monitorPeriod, + String alarmLevel, + DataAcquisitionStrategy daStrategy, + AnomalyDetectionStrategy adStrategy, + TaskTriggerStrategy ttStrategy) { + this(name, alais, description, Long.parseLong(monitorPeriod), + MonitorStrategyAttachedTaskAdapter.getAlarmLevelValueFromString(alarmLevel), + daStrategy, adStrategy, ttStrategy); + } + + public class DataAcquisitionStrategy { + + public static final String ALL_ITEMS_CONF_CONSTANT = "#ALL ITEMS#"; + public static final String NO_ITEM_CONF_CONSTANT = "#NO ITEM#"; + + class DataAcquisitionMetaStrategy { + + private String monitorItemName = null; + private long timeout = 0; // no timeout limit by default + private List attachedTaskConfs = null; + + public DataAcquisitionMetaStrategy( + String monitorItemName, + long timeout, + List attachedTaskConfs) { + this.monitorItemName = monitorItemName; + this.timeout = timeout; + this.attachedTaskConfs = attachedTaskConfs; + } + + public String getMonitorItemName() { + return this.monitorItemName; + } + + public long getTimeout() { + return this.timeout; + } + + public List getAttachedTaskConfs() { + return attachedTaskConfs; + } + } + + /** + * MonitorItemName <--> Data Acquisition Meta-Strategy + */ + private Map metas = + new HashMap(); + + /** + * MonitorItemName <--> Attached Tasks triggered when timeout of + * data acquisition expires. + */ + private Map> + attachedTaskAdaptersMap = + new HashMap>(); + + + public DataAcquisitionStrategy() { + // nothing needed to do + } + + public void putMeta(String monitorItemName, long timeout, + List attachedTaskConfs) { + metas.put(monitorItemName, + new DataAcquisitionMetaStrategy( + monitorItemName, timeout, attachedTaskConfs)); + } + + public void putMeta(String monitorItemName, String timeout, + List attachedTaskConfs) { + putMeta(monitorItemName, Long.parseLong(timeout), attachedTaskConfs); + } + + public long getTimeoutMetaByMonitorItemName(String monitorItemName) { + return metas.get(monitorItemName).getTimeout(); + } + + public Set getAllInvolvedItemNames() { + return metas.keySet(); + } + + public int getAllInvolvedItemNumber() { + return metas.size(); + } + + public boolean containsMonitorItem(String monitorItemName) { + return metas.containsKey(monitorItemName); + } + + public List getAttachedTaskConfsByMonitorItemName(String monitorItemName){ + return metas.get(monitorItemName).getAttachedTaskConfs(); + } + + private void generateAttachedTaskAdapters() { + attachedTaskAdaptersMap = + new HashMap>(); + Iterator> iter = + metas.entrySet().iterator(); + while (iter.hasNext()) { + + Map.Entry entry = + iter.next(); + String monitorItemName = entry.getKey(); + DataAcquisitionMetaStrategy meta = entry.getValue(); + + List adapterList = + new ArrayList(); + List attachedTaskConfs = meta.getAttachedTaskConfs(); + for (Element conf : attachedTaskConfs) { + MonitorStrategyAttachedTaskAdapter adapter = + MonitorStrategyAttachedTaskAdapterGenerater + .generateMonitorStrategyAttachedTaskAdapter(conf); + + if (adapter == null) { + LOG.warn("Can't generate attached task adapter from conf: " + + conf); + continue; + } + + adapterList.add(adapter); + } + + /* + * 'adapterList' will have no elements if there are no attached + * tasks configured for relevant acquired monitor item. + */ + attachedTaskAdaptersMap.put(monitorItemName, adapterList); + } + + } + + public List + getAttachedTaskAdaptersByMonitorItemName( + String monitorItemName) { + if (attachedTaskAdaptersMap.size() == 0) { + generateAttachedTaskAdapters(); + } + return attachedTaskAdaptersMap.get(monitorItemName); + } + } + + + class AnomalyDetectionStrategy { + + class AnomalyDetectionMetaStrategy { + + private String monitorItemName = null; + private String monitorDataTypeName = null; + private String monitorDataSubTypeName = null; + private Element strategy = null; + + public AnomalyDetectionMetaStrategy( + String monitorItemName, + String monitorDataTypeName, + String monitorDataSubTypeName, + Element strategy) { + this.monitorItemName = monitorItemName; + this.monitorDataTypeName = monitorDataTypeName; + this.monitorDataSubTypeName = monitorDataSubTypeName; + this.strategy = strategy; + } + + /** + * @return the monitorItemName + */ + public String getMonitorItemName() { + return monitorItemName; + } + + /** + * @return the monitorDataTypeName + */ + public String getMonitorDataTypeName() { + return monitorDataTypeName; + } + + /** + * @return the monitorDataSubTypeName + */ + public String getMonitorDataSubTypeName() { + return monitorDataSubTypeName; + } + + /** + * @return the strategy + */ + public Element getStrategy() { + return strategy; + } + + } + + private Map metas = + new HashMap(); + + private List detectors = + new ArrayList(); + + public AnomalyDetectionStrategy() { + // nothing needed to do + } + + public void putMeta(String monitorItemName, + String monitorDataTypeName, + String monitorDataSubTypeName, + Element strategy) { + metas.put(monitorItemName, + new AnomalyDetectionMetaStrategy( + monitorItemName, monitorDataTypeName, + monitorDataSubTypeName, strategy)); + } + + /** + * This method should be called whenever all involved metas + * have been inserted. + */ + private void generateAnomalyDetectors() { + detectors = new ArrayList(); + Iterator> iter = + metas.entrySet().iterator(); + while (iter.hasNext()) { + AnomalyDetectionMetaStrategy meta = iter.next().getValue(); + AnomalyDetector detector = + AnomalyDetectorGenerator.generate( + meta.getMonitorItemName(), + meta.getMonitorDataTypeName(), + meta.getMonitorDataSubTypeName(), + meta.getStrategy()); + if (detector == null) { + LOG.warn("Can't generate anomaly detector from conf: " + + meta); + continue; + } + detectors.add(detector); + } + } + + public List getAllInvolvedAnomalyDetectors() { + if (detectors.size() == 0) { + generateAnomalyDetectors(); + } + return detectors; + } + } + + + class TaskTriggerStrategy { + + private String mode = null; + private Element strategy = null; + private List attachedTaskConfs = null; + + private TaskTrigger visitor = null; + + private List + attachedTaskAdapters = + new ArrayList(); + + public TaskTriggerStrategy(String mode, Element strategy, + List attachedTaskConfs) { + this.mode = mode; + this.strategy = strategy; + this.attachedTaskConfs = attachedTaskConfs; + + this.generateTaskTrigger(); + this.generateAttachedTaskAdapters(); + } + + private void generateTaskTrigger() { + try { + visitor = (TaskTrigger) Class.forName( + "hma.monitor.strategy.trigger." + + mode + + "TypeTaskTrigger").getConstructor( + Element.class).newInstance(strategy); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + public TaskTrigger getTaskTrigger() { + return this.visitor; + } + + private void generateAttachedTaskAdapters() { + + attachedTaskAdapters = + new ArrayList(); + + for (Element conf : attachedTaskConfs) { + MonitorStrategyAttachedTaskAdapter adapter = + MonitorStrategyAttachedTaskAdapterGenerater + .generateMonitorStrategyAttachedTaskAdapter(conf); + if (adapter == null) { + LOG.warn("Can't generate attached task adapter from conf: " + + conf); + continue; + } + attachedTaskAdapters.add(adapter); + } + + } + + public List getAttachedTaskAdapters() { + return this.attachedTaskAdapters; + } + + } + + + + /** + * @param daStrategy the daStrategy to set + */ + public void setDataAcquisitionStrategy(DataAcquisitionStrategy daStrategy) { + this.daStrategy = daStrategy; + } + + /** + * @return the daStrategy + */ + public DataAcquisitionStrategy getDataAcquisitionStrategy() { + return daStrategy; + } + + /** + * @param adStrategy the adStrategy to set + */ + public void setAnomalyDetectionStrategy(AnomalyDetectionStrategy adStrategy) { + this.adStrategy = adStrategy; + } + + /** + * @return the adStrategy + */ + public AnomalyDetectionStrategy getAnomalyDetectionStrategy() { + return adStrategy; + } + + /** + * @param ttStrategy the ttStrategy to set + */ + public void setTaskTriggerStrategy(TaskTriggerStrategy ttStrategy) { + this.ttStrategy = ttStrategy; + } + + /** + * @return the ttStrategy + */ + public TaskTriggerStrategy getTaskTriggerStrategy() { + return ttStrategy; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param alias the alias to set + */ + public void setAlias(String alias) { + this.alias = alias; + } + + /** + * @return the alias + */ + public String getAlias() { + return alias; + } + + /** + * @param description the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param monitorPeriod the monitorPeriod to set + */ + public void setMonitorPeriod(long monitorPeriod) { + this.monitorPeriod = monitorPeriod; + } + + /** + * @return the monitorPeriod + */ + public long getMonitorPeriod() { + return monitorPeriod; + } + + /** + * @param alarmLevel the alarmLevel to set + */ + public void setAlarmLevel(int alarmLevel) { + this.alarmLevel = alarmLevel; + } + + /** + * @return the alarmLevel + */ + public int getAlarmLevel() { + return alarmLevel; + } + +} diff --git a/src/java/hma/monitor/strategy/MonitorStrategyWorker.java b/src/java/hma/monitor/strategy/MonitorStrategyWorker.java new file mode 100644 index 0000000..af3abd6 --- /dev/null +++ b/src/java/hma/monitor/strategy/MonitorStrategyWorker.java @@ -0,0 +1,624 @@ +/** + * + */ +package hma.monitor.strategy; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.monitor.MonitorStrategy; +import hma.monitor.collection.MonitorItemData; +import hma.monitor.strategy.MonitorStrategyPrototype.AnomalyDetectionStrategy; +import hma.monitor.strategy.MonitorStrategyPrototype.DataAcquisitionStrategy; +import hma.monitor.strategy.MonitorStrategyPrototype.TaskTriggerStrategy; +import hma.monitor.strategy.detect.AnomalyDetectionResult; +import hma.monitor.strategy.detect.AnomalyDetector; +import hma.monitor.strategy.trigger.TaskTrigger; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskArguments; +import hma.util.HMALogger; +import hma.util.LOG; +import hma.util.TimedTask; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; + +import org.json.JSONObject; +import org.json.JSONArray; +import org.json.JSONException; + +/** + * @author guoyezhi + * add additionalInfo by chiwen01 + * + */ +public class MonitorStrategyWorker extends TimedTask implements Runnable { + + private static String CLUSTER_NAME = null; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd" + " " + "HH:mm:ss"); + + private MonitorStrategy strategy = null; + + private String strategyName = null; + + private Date currentDataAcquisitionTimePoint = null; + private long firstDataAcquisitionTimestamp = -1; + private String detectionTimestampStr = null; + + private HMALogger logger = null; + //private HMALogger adLogger = null; + + + private String dbServer = null; + private String dbName = null; + private String resultTableName = null; + private String infoTableName = null; + private String userName = null; + private String password = null; + + private Connection dbConn = null; + //private Connection infTableConn = null; + + + public MonitorStrategyWorker(MonitorStrategy strategy) throws SQLException { + CLUSTER_NAME = MonitorManager.getMonitoredClusterName(); + this.strategy = strategy; + this.strategyName = this.strategy.getPrototype().getName(); + this.defaultAlarmLevel = strategy.getPrototype().getAlarmLevel(); + this.logger = new HMALogger(strategyName + ".log"); + //this.adLogger = new HMALogger(strategyName + ".ad.log"); + this.initConnection(); + } + + private void initConnection() throws SQLException { + + Configuration hmaConf = + MonitorManager.getMonitorManager().getConf(); + + dbServer = hmaConf.get( + "monitor.data.db.server", "yf-2950-016.yf01.baidu.com"); + dbName = hmaConf.get( + "monitor.data.db.name", "hma"); + resultTableName = hmaConf.get( + "monitor.result.data.db.table", "monitor_result"); + infoTableName = hmaConf.get( + "monitor.info.data.db.table", "monitor_info"); + userName = hmaConf.get( + "monitor.db.user.name", "root"); + password = hmaConf.get( + "monitor.db.user.pass", "hmainit"); + + /* + dbConn = DriverManager.getConnection( + "jdbc:mysql://" + dbServer + "/" + dbName, + userName, + password); + */ + dbConn = DriverManager.getConnection( + "jdbc:mysql://" + dbServer + "/" + dbName + "?user=" + userName + "&password=" + password + "&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false"); + + /* + dbConn = DriverManager.getConnection( + "jdbc:mysql://" + dbServer + "/" + dbName + + "?useUnicode=true&characterEncoding=utf8", + userName, + password); + */ + } + + + /** + * @param strategy the strategy to set + */ + public void setStrategy(MonitorStrategy strategy) { + this.strategy = strategy; + } + + /** + * @return the strategy + */ + public MonitorStrategy getStrategy() { + return strategy; + } + + /** + * @param currentDataAcquisitionTimePoint the currentDataAcquisitionTimePoint to set + */ + public void setDataAcquisitionTimepoint(Date dataAcquisitionTimePoint) { + this.currentDataAcquisitionTimePoint = dataAcquisitionTimePoint; + } + + /** + * @return the currentDataAcquisitionTimePoint + */ + public Date getDataAcquisitionTimepoint() { + return currentDataAcquisitionTimePoint; + } + + @SuppressWarnings("unchecked") + private boolean acquireData() { + + long currentTime = System.currentTimeMillis(); + setDataAcquisitionTimepoint(new Date(currentTime)); + if (firstDataAcquisitionTimestamp < 0) { + firstDataAcquisitionTimestamp = currentTime; + } + logger.log("[" + strategyName + "] : DataAcquisitionTimepoint: " + + getDataAcquisitionTimepoint()); + + Map> dataPool = getStrategy().getDataPool(); + NavigableMap dataProcessingCentre = + getStrategy().getDataProcessingCentre(); + + MonitorStrategyPrototype prototype = getStrategy().getPrototype(); + DataAcquisitionStrategy daStrategy = prototype.getDataAcquisitionStrategy(); + + MonitorStrategyDataCollection dataCollection = + new MonitorStrategyDataCollection(prototype.getName()); + boolean acqFailed = false; + String fullInfo = null; + StringBuilder fullInfoBuilder = new StringBuilder(); + + if (daStrategy.containsMonitorItem(DataAcquisitionStrategy.ALL_ITEMS_CONF_CONSTANT)) { + + long timeout = + daStrategy.getTimeoutMetaByMonitorItemName( + DataAcquisitionStrategy.ALL_ITEMS_CONF_CONSTANT); + + List adapterList = + daStrategy.getAttachedTaskAdaptersByMonitorItemName( + DataAcquisitionStrategy.ALL_ITEMS_CONF_CONSTANT); + + StringBuilder resInfoBuilder = new StringBuilder(); + JSONObject timeoutItemNameJsonObject = new JSONObject(); + + synchronized (dataPool) { + Iterator iter = dataPool.keySet().iterator(); + while (iter.hasNext()) { + + String itemName = iter.next(); + + long timestamp = -1; + + MonitorItemData itemData = dataPool.get(itemName); + if (itemData == null) { + /* + * If we get itemData == null here, it indicate relevant + * collector/extractor hasn't got data ready for detection! + * + * To prevent this situation lasting for a long time, we use + * a fake data collection timestamp -- firstDataAcquisitionTimestamp. + */ + timestamp = firstDataAcquisitionTimestamp; + } else { + timestamp = itemData.getTimestamp(); + } + + if ((timeout > 0) && (currentTime - timestamp >= timeout)) { + acqFailed = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + resInfoBuilder.append( + "\t监控项 " + itemName + " 数据采集超时 " + + ((currentTime - timestamp)) / 1000 + " 秒\n"); + try { + timeoutItemNameJsonObject.put("additionalInfo", itemData.getAdditionalInfo()); + } catch (JSONException e) { + e.getCause(); + } + } else if (itemData == null) { + // nothing needed to do + } else { + dataCollection.put(itemName, + new MonitorStrategyData(itemData)); + } + } + } + + if (acqFailed == true) { + + if (resInfoBuilder.length() > 0) { + + MonitorStrategyAttachedTaskArguments args = + new MonitorStrategyAttachedTaskArguments( + CLUSTER_NAME, + strategyName, + MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue(alarmLevel), + "复合型监控项数据采集超时", + "复合型监控项数据采集超时 :\n" + resInfoBuilder.toString(), + getDataAcquisitionTimepoint(), + timeoutItemNameJsonObject.toString()); + + // trigger attached tasks + for (MonitorStrategyAttachedTaskAdapter adapter : adapterList) { + adapter.runTask(args); + } + + fullInfo = "复合型监控项数据采集超时 :\n" + resInfoBuilder.toString(); + } + + } else { + for (MonitorStrategyAttachedTaskAdapter adapter : adapterList) { + adapter.clearContinuousOccurrence(); + } + } + } + else { + + Iterator iter = + daStrategy.getAllInvolvedItemNames().iterator(); + + while (iter.hasNext()) { + + String itemName = iter.next(); + + if (itemName.equals(DataAcquisitionStrategy.NO_ITEM_CONF_CONSTANT)) { + /* + * If we get "#NO ITEM#" here, just skip it + */ + continue; + } + + long timeout = + daStrategy.getTimeoutMetaByMonitorItemName(itemName); + List adapterList = + daStrategy.getAttachedTaskAdaptersByMonitorItemName(itemName); + + long timestamp = -1; + MonitorItemData itemData = null; + + synchronized (dataPool) { + itemData = dataPool.get(itemName); + } + if (itemData == null) { + /* + * If we get itemData == null here, it indicate relevant + * collector/extractor hasn't got data ready for detection! + * + * To prevent this situation lasting for a long time, we use + * a fake data collection timestamp -- firstDataAcquisitionTimestamp. + */ + timestamp = firstDataAcquisitionTimestamp; + } else { + timestamp = itemData.getTimestamp(); + } + if ((timeout > 0) && (currentTime - timestamp >= timeout)) { + + acqFailed = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + + MonitorStrategyAttachedTaskArguments args = + new MonitorStrategyAttachedTaskArguments( + CLUSTER_NAME, + strategyName, + MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue(alarmLevel), + "监控项 " + itemName + " 数据采集超时 " + ((currentTime - timestamp)) / 1000 + " 秒", + "监控项 " + itemName + " 数据采集超时 " + ((currentTime - timestamp)) / 1000 + " 秒", + getDataAcquisitionTimepoint(), + itemData.getAdditionalInfo()); + + // trigger attached tasks + for (MonitorStrategyAttachedTaskAdapter adapter : adapterList) { + adapter.runTask(args); + } + + fullInfoBuilder.append( + "监控项 " + itemName + " 数据采集超时 " + ((currentTime - timestamp)) / 1000 + " 秒\n"); + + } else if (itemData == null) { + // nothing needed to do + } else { + for (MonitorStrategyAttachedTaskAdapter adapter : adapterList) { + adapter.clearContinuousOccurrence(); + } + dataCollection.put(itemName, new MonitorStrategyData(itemData)); + } + } + + if (acqFailed == true) { + if (daStrategy.getAllInvolvedItemNames().size() > 1) { + fullInfo = "组合型监控项数据采集超时 :\n" + fullInfoBuilder.toString(); + } else { + fullInfo = fullInfoBuilder.toString(); + } + } + + } + + detectionTimestampStr = dateFormat.format(new Date()); + + if (acqFailed == true) { + String levelStr = + MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue( + alarmLevel); + repositMonitorResult(detectionTimestampStr, levelStr, 1); + repositMonitorAlarmInfo(detectionTimestampStr, levelStr, fullInfo); + return false; + } + + synchronized (dataProcessingCentre) { + dataProcessingCentre.put( + this.getDataAcquisitionTimepoint(), dataCollection); + } + + return true; + } + + + private void repositMonitorResult( + String detectionTimestamp, + String strategyLevel, + int detectionResult) { + + PreparedStatement pstat = null; + try { + pstat = dbConn.prepareStatement( + "INSERT INTO " + resultTableName + " VALUES ( " + + "'" + CLUSTER_NAME + "', " + + "'" + strategyName + "', " + + "?, ?, ? )"); + pstat.setString(1, detectionTimestamp); + pstat.setString(2, strategyLevel); + pstat.setInt(3, detectionResult); + pstat.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } finally { + if (pstat != null) { + try { + pstat.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + } + + private void repositMonitorAlarmInfo( + String detectionTimestamp, + String infoLevel, + String infoContent) { + + PreparedStatement pstat = null; + try { + pstat = dbConn.prepareStatement( + "INSERT INTO " + infoTableName + " VALUES ( " + + "'" + CLUSTER_NAME + "', " + + "'" + strategyName + "', " + + "?, ?, ? )"); + pstat.setString(1, detectionTimestamp); + pstat.setString(2, infoLevel); + pstat.setString(3, infoContent); + System.out.println(pstat.toString()); + pstat.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } finally { + if (pstat != null) { + try { + pstat.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + } + + + private int defaultAlarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_UNINIT; + private int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_UNINIT; + private List detectResults = null; + + private void detectAnomaly() { + + AnomalyDetectionStrategy adStrategy = + getStrategy().getPrototype().getAnomalyDetectionStrategy(); + + boolean anomalous = true; + + + List detectors = + adStrategy.getAllInvolvedAnomalyDetectors(); + for (AnomalyDetector detector : detectors) { + AnomalyDetectionResult res = detector.detect(this); + /* + * If return null, it indicates that the detection result is + * not needed. + */ + if (res == null) { + anomalous = false; + continue; + } + + detectResults.add(res); + if (res.isAnomalous() == false) { + anomalous = false; + continue; + } + int newLevel = res.getAlarmLevel(); + if (alarmLevel < newLevel) + alarmLevel = newLevel; + } + + if (anomalous == true) { + NavigableMap dataProcessingCentre = + getStrategy().getDataProcessingCentre(); + synchronized (dataProcessingCentre) { + dataProcessingCentre.get( + getDataAcquisitionTimepoint()).setAnomalous(anomalous); + } + } + + } + + + private void triggerAttachedTasks() { + + TaskTriggerStrategy ttStrategy = + getStrategy().getPrototype().getTaskTriggerStrategy(); + + TaskTrigger taskTrigger = ttStrategy.getTaskTrigger(); + + //System.out.println("mode = " + mode); + //System.out.println("strategyConf = " + strategyConf); + + List adapterList = + ttStrategy.getAttachedTaskAdapters(); + + if (taskTrigger.visitMonitorStrategyWorker(this) == true) { + + String keyInfo = null; + StringBuilder keyInfoBuilder = new StringBuilder(); + String fullInfo = null; + StringBuilder fullInfoBuilder = new StringBuilder(); + String additionalInfo = null; + StringBuilder additionalInfoBuilder = new StringBuilder(); + + /* + * TODO: keyInfo = prototype.getKeyAlarmMessage(); + */ + for (AnomalyDetectionResult res : detectResults) { + + String alarmInfo = res.getAlarmInfo(); + int index = alarmInfo.indexOf("\n"); + if (index > 0) + alarmInfo = alarmInfo.substring(0, index); + keyInfoBuilder.append(alarmInfo + " "); + + fullInfoBuilder.append(res.getAlarmInfo()); + + additionalInfoBuilder.append(res.getAdditionalInfo()); + } + if (keyInfo == null) + keyInfo = keyInfoBuilder.toString(); + fullInfo = fullInfoBuilder.toString(); + additionalInfo = additionalInfoBuilder.toString(); + + MonitorStrategyAttachedTaskArguments args = + new MonitorStrategyAttachedTaskArguments( + CLUSTER_NAME, + strategyName, + MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue( + alarmLevel), + keyInfo, + fullInfo, + new Date(), + additionalInfo); + + for (MonitorStrategyAttachedTaskAdapter adapter : adapterList) { + adapter.runTask(args); + } + + String levelStr = + MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue( + alarmLevel); + repositMonitorResult(detectionTimestampStr, levelStr, 1); + repositMonitorAlarmInfo(detectionTimestampStr, levelStr, fullInfo); + + } else { + for (MonitorStrategyAttachedTaskAdapter adapter : adapterList) { + adapter.clearContinuousOccurrence(); + } + + String levelStr = + MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue( + alarmLevel); + repositMonitorResult(detectionTimestampStr, levelStr, 0); + + } // if (taskTrigger.visitMonitorStrategyWorker(this) == true) + + } + + + private static final int DATA_INMEM_MAX_CYCLE = 100; + + private void resizeDataProcessingCentre() { + NavigableMap dataProcessingCentre = + getStrategy().getDataProcessingCentre(); + synchronized (dataProcessingCentre) { + while (dataProcessingCentre.size() > DATA_INMEM_MAX_CYCLE) { + dataProcessingCentre.pollFirstEntry(); + } + } + } + + + /* (non-Javadoc) + * @see hma.util.TimedTask#run() + */ + @Override + public synchronized void run() { + + alarmLevel = defaultAlarmLevel; + detectResults = new ArrayList(); + + //LOG.debug("[" + strategyName + "] : acquiring data ..."); + logger.log("[" + strategyName + "] : acquiring data ..."); + if (this.acquireData() == false) + return; + + LOG.debug("[" + strategyName + "] : detecting anomaly ..."); + logger.log("[" + strategyName + "] : detecting anomaly ..."); + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_UNINIT; + detectAnomaly(); + + //LOG.debug("[" + strategyName + "] : triggering tasks ..."); + logger.log("[" + strategyName + "] : triggering tasks ..."); + triggerAttachedTasks(); + + logger.log("[" + strategyName + "] : resizing data processing centre ..."); + resizeDataProcessingCentre(); + + //LOG.debug("************************************************************"); + logger.log("************************************************************"); + + } + + + /** + * @param args + */ + public static void main(String[] args) { + + String name = "SimulationMonitorStrategy"; + + MonitorStrategyConfiguration conf = new MonitorStrategyConfiguration(); + MonitorStrategyPrototype prototype = conf.getAllPrototypes().get(name); + System.out.println(prototype); + + MonitorStrategy strategy = new MonitorStrategy(prototype, null); + + LOG.warn("---> Anomaly Detectors:"); + AnomalyDetectionStrategy adStrategy = + strategy.getPrototype().getAnomalyDetectionStrategy(); + List detectors = + adStrategy.getAllInvolvedAnomalyDetectors(); + for (AnomalyDetector detector : detectors) { + LOG.warn(detector + ""); + } + + TaskTriggerStrategy ttStrategy = + strategy.getPrototype().getTaskTriggerStrategy(); + TaskTrigger visitor = ttStrategy.getTaskTrigger(); + System.out.println(visitor); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/AnomalyDetectionResult.java b/src/java/hma/monitor/strategy/detect/AnomalyDetectionResult.java new file mode 100644 index 0000000..3d52481 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/AnomalyDetectionResult.java @@ -0,0 +1,70 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * add additionalInfo by chiwen01 + * + */ +public class AnomalyDetectionResult { + + private boolean anomalous = true; + + private int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_UNINIT; + + private String alarmInfo = null; + + private String dectionInfo = null; + + private String additionalInfo = null; + + + public AnomalyDetectionResult( + boolean anomalous, + int alarmLevel, + String alarmInfo, + String dectionInfo) { + this.anomalous = anomalous; + this.alarmLevel = alarmLevel; + this.alarmInfo = alarmInfo; + this.dectionInfo = dectionInfo; + } + + public AnomalyDetectionResult( + boolean anomalous, + int alarmLevel, + String alarmInfo, + String dectionInfo, + String additionalInfo) { + this.anomalous = anomalous; + this.alarmLevel = alarmLevel; + this.alarmInfo = alarmInfo; + this.dectionInfo = dectionInfo; + this.additionalInfo = additionalInfo; + } + + public boolean isAnomalous() { + return anomalous; + } + + public int getAlarmLevel() { + return alarmLevel; + } + + public String getAlarmInfo() { + return alarmInfo; + } + + public String getDectionInfo() { + return dectionInfo; + } + + public String getAdditionalInfo() { + return additionalInfo; + } +} diff --git a/src/java/hma/monitor/strategy/detect/AnomalyDetector.java b/src/java/hma/monitor/strategy/detect/AnomalyDetector.java new file mode 100644 index 0000000..09882f3 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/AnomalyDetector.java @@ -0,0 +1,26 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyWorker; + +/** + * Note: The whole module hierarchy is designed under Strategy + * Design Pattern. + * + * @author guoyezhi + * + */ +public interface AnomalyDetector { + + /** + * If return null, it indicates that the detection result is + * not needed. + * + * @param worker + * @return + */ + AnomalyDetectionResult detect(MonitorStrategyWorker worker); + +} diff --git a/src/java/hma/monitor/strategy/detect/AnomalyDetectorGenerator.java b/src/java/hma/monitor/strategy/detect/AnomalyDetectorGenerator.java new file mode 100644 index 0000000..743c19c --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/AnomalyDetectorGenerator.java @@ -0,0 +1,739 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.util.LOG; + +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; + +/** + * @author guoyezhi + * + */ +public class AnomalyDetectorGenerator { + + @SuppressWarnings("unchecked") + private static SingleCycleAnomalyDetector generateSingleCycleAnomalyDetector( + String monitorItemName, + String monitorDataTypeName, + String monitorDataSubTypeName, + Element strategy) { + + if (monitorItemName.equals("SingleCycle") == false) { + // TODO: + } + + SingleCycleAnomalyDetector singleCycleDetector = null; + + if (monitorDataSubTypeName.equals("SingleLong")) { + + String conditionConf = null; + String thresholdConf = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("condition".equals(subField.getTagName()) && subField.hasChildNodes()) + conditionConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("threshold".equals(subField.getTagName()) && subField.hasChildNodes()) + thresholdConf = ((Text)subField.getFirstChild()).getData().trim(); + } + + if (conditionConf != null && thresholdConf != null) { + + LOG.warn("conditionConf = " + conditionConf); + LOG.warn("thresholdConf = " + thresholdConf); + + long threshold = Long.parseLong(thresholdConf); + + Class detectorClass = null; + try { + LOG.warn("hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + + detectorClass = + (Class) Class.forName( + "hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + try { + singleCycleDetector = + detectorClass.getConstructor( + String.class, Long.class).newInstance( + monitorItemName, threshold); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + } + else if (monitorDataSubTypeName.equals("SingleDouble")) { + + String conditionConf = null; + String thresholdConf = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("condition".equals(subField.getTagName()) && subField.hasChildNodes()) + conditionConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("threshold".equals(subField.getTagName()) && subField.hasChildNodes()) + thresholdConf = ((Text)subField.getFirstChild()).getData().trim(); + } + + if (conditionConf != null && thresholdConf != null) { + + double threshold = Double.parseDouble(thresholdConf); + + Class detectorClass = null; + try { + detectorClass = + (Class) Class.forName( + "hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + try { + singleCycleDetector = + detectorClass.getConstructor( + String.class, Double.class).newInstance( + monitorItemName, threshold); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + } + else if (monitorDataSubTypeName.equals("SingleString")) { + + String conditionConf = null; + String patternConf = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("condition".equals(subField.getTagName()) && subField.hasChildNodes()) + conditionConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("pattern".equals(subField.getTagName()) && subField.hasChildNodes()) + patternConf = ((Text)subField.getFirstChild()).getData().trim(); + } + + //if (conditionConf != null && patternConf != null) { + if (conditionConf != null) { + + LOG.warn("conditionConf = " + conditionConf); + LOG.warn("patternConf = " + patternConf); + + Class detectorClass = null; + try { + LOG.warn("hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + + detectorClass = + (Class) Class.forName( + "hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + try { + singleCycleDetector = + detectorClass.getConstructor( + String.class, String.class).newInstance( + monitorItemName, patternConf); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + } + else if (monitorDataSubTypeName.equals("CustomedSingleString")) { + + //System.out.println("monitorItemName = " + monitorItemName); + //System.out.println("monitorDataTypeName = " + monitorDataTypeName); + //System.out.println("monitorDataSubTypeName = " + monitorDataSubTypeName); + + String patternConf = null; + String detectorConf = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("pattern".equals(subField.getTagName()) && subField.hasChildNodes()) + patternConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("detector".equals(subField.getTagName()) && subField.hasChildNodes()) { + detectorConf = ((Text)subField.getFirstChild()).getData().trim(); + } + } + + LOG.warn("patternConf = " + patternConf); + LOG.warn("detectorConf = " + detectorConf); + + if (detectorConf != null) { + + //long threshold = Long.parseLong(thresholdConf); + + Class detectorClass = null; + try { + detectorClass = + (Class) Class.forName( + detectorConf); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + + if (detectorClass != null) { + + try { + singleCycleDetector = + detectorClass.getConstructor( + String.class, String.class).newInstance( + monitorItemName, patternConf); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + } else { + System.out.println("!!"); + } + + } + + } + else if (monitorDataSubTypeName.equals("Compound")) { + + //System.out.println("monitorItemName = " + monitorItemName); + //System.out.println("monitorDataTypeName = " + monitorDataTypeName); + //System.out.println("monitorDataSubTypeName = " + monitorDataSubTypeName); + + String detectorConf = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("detector".equals(subField.getTagName()) && subField.hasChildNodes()) { + detectorConf = ((Text)subField.getFirstChild()).getData().trim(); + } + } + + //LOG.warn("detectorConf = " + detectorConf); + + if (detectorConf != null) { + + //long threshold = Long.parseLong(thresholdConf); + + Class detectorClass = null; + try { + detectorClass = + (Class) Class.forName( + detectorConf); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + + if (detectorClass != null) { + + try { + singleCycleDetector = + detectorClass.getConstructor( + String.class).newInstance(monitorItemName); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + } else { + System.out.println("!!"); + } + + } + + } + else { + throw new RuntimeException( + "Unknown Single Cycle Anomaly Detector Sub-Type"); + } + + return singleCycleDetector; + } + + @SuppressWarnings("unchecked") + private static DoubleCyclesAnomalyDetector generateDoubleCyclesAnomalyDetector( + String monitorItemName, + String monitorDataTypeName, + String monitorDataSubTypeName, + Element strategy) { + + if (monitorItemName.equals("DoubleCycles") == false) { + // TODO: + } + + DoubleCyclesAnomalyDetector doubleCyclesDetector = null; + + if (monitorDataSubTypeName.equals("SingleLong")) { + + String conditionConf = null; + String thresholdConf = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("condition".equals(subField.getTagName()) && subField.hasChildNodes()) + conditionConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("threshold".equals(subField.getTagName()) && subField.hasChildNodes()) + thresholdConf = ((Text)subField.getFirstChild()).getData().trim(); + } + + if (conditionConf != null && thresholdConf != null) { + + LOG.warn("conditionConf = " + conditionConf); + LOG.warn("thresholdConf = " + thresholdConf); + + long threshold = Long.parseLong(thresholdConf); + + Class detectorClass = null; + try { + LOG.warn("hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + + detectorClass = + (Class) Class.forName( + "hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + try { + doubleCyclesDetector = + detectorClass.getConstructor( + String.class, Long.class).newInstance( + monitorItemName, threshold); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + } + else if (monitorDataSubTypeName.equals("SingleDouble")) { + + String conditionConf = null; + String thresholdConf = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("condition".equals(subField.getTagName()) && subField.hasChildNodes()) + conditionConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("threshold".equals(subField.getTagName()) && subField.hasChildNodes()) + thresholdConf = ((Text)subField.getFirstChild()).getData().trim(); + } + + if (conditionConf != null && thresholdConf != null) { + + LOG.warn("conditionConf = " + conditionConf); + LOG.warn("thresholdConf = " + thresholdConf); + + double threshold = Double.parseDouble(thresholdConf); + + Class detectorClass = null; + try { + LOG.warn("hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + + detectorClass = + (Class) Class.forName( + "hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + try { + doubleCyclesDetector = + detectorClass.getConstructor( + String.class, Double.class).newInstance( + monitorItemName, threshold); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + } + else if (monitorDataSubTypeName.equals("Compound")) { + + String detectorConf = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("detector".equals(subField.getTagName()) && subField.hasChildNodes()) { + detectorConf = ((Text)subField.getFirstChild()).getData().trim(); + } + } + + if (detectorConf != null) { + + //long threshold = Long.parseLong(thresholdConf); + + Class detectorClass = null; + try { + detectorClass = + (Class) Class.forName( + detectorConf); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + + if (detectorClass != null) { + + try { + doubleCyclesDetector = + detectorClass.getConstructor( + String.class).newInstance(monitorItemName); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + } else { + System.out.println("!!"); + } + + } + + } + else { + throw new RuntimeException( + "Unknown Double Cycles Anomaly Detector Sub-Type"); + } + + return doubleCyclesDetector; + } + + @SuppressWarnings("unchecked") + private static MultipleCyclesMapTypeAnomalyDetector + generateMultipleCyclesMapTypeAnomalyDetector( + String monitorItemName, + String monitorDataTypeName, + String monitorDataSubTypeName, + Element strategy) { + + MultipleCyclesMapTypeAnomalyDetector multipleCyclesMapDetector = null; + + if (monitorDataSubTypeName.equals("LongMap")) { + + String conditionConf = null; + String cyclesConf = null; + String thresholdConf = "0"; + Element subStrategy = null; + + SingleCycleSingleLongAnomalyDetector singleLongDetector = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("condition".equals(subField.getTagName()) && subField.hasChildNodes()) + conditionConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("cycles".equals(subField.getTagName()) && subField.hasChildNodes()) + cyclesConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("threshold".equals(subField.getTagName()) && subField.hasChildNodes()) + thresholdConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("sub-strategy".equals(subField.getTagName()) && subField.hasChildNodes()) + subStrategy = subField; + } + + if (conditionConf != null && cyclesConf != null + && thresholdConf != null && subStrategy != null) { + + LOG.warn("monitorItemName = " + monitorItemName); + + singleLongDetector = (SingleCycleSingleLongAnomalyDetector) generateSingleCycleAnomalyDetector( + monitorItemName, "SingleCycle", "SingleLong", subStrategy); + + if (singleLongDetector != null) { + Class detectorClass = null; + try { + detectorClass = + (Class) Class.forName( + "hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + int cycles = Integer.parseInt(cyclesConf); + int threshold = Integer.parseInt(thresholdConf); + + try { + multipleCyclesMapDetector = + detectorClass.getConstructor(String.class, Integer.class, + Integer.class, SingleCycleSingleLongAnomalyDetector.class).newInstance( + monitorItemName, cycles, threshold, + singleLongDetector); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + } + + } + else if (monitorDataSubTypeName.equals("DoubleMap")) { + + String conditionConf = null; + String cyclesConf = null; + String thresholdConf = "0"; + Element subStrategy = null; + + SingleCycleSingleDoubleAnomalyDetector singleDoubleDetector = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("condition".equals(subField.getTagName()) && subField.hasChildNodes()) + conditionConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("cycles".equals(subField.getTagName()) && subField.hasChildNodes()) + cyclesConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("threshold".equals(subField.getTagName()) && subField.hasChildNodes()) + thresholdConf = ((Text)subField.getFirstChild()).getData().trim(); + if ("sub-strategy".equals(subField.getTagName()) && subField.hasChildNodes()) + subStrategy = subField; + } + + if (conditionConf != null && cyclesConf != null + && thresholdConf != null && subStrategy != null) { + + singleDoubleDetector = (SingleCycleSingleDoubleAnomalyDetector) generateSingleCycleAnomalyDetector( + monitorItemName, "SingleCycle", "SingleDouble", subStrategy); + + if (singleDoubleDetector != null) { + Class detectorClass = null; + try { + detectorClass = + (Class) Class.forName( + "hma.monitor.strategy.detect." + + monitorDataTypeName + + monitorDataSubTypeName + + conditionConf + + "AnomalyDetector"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + int cycles = Integer.parseInt(cyclesConf); + int threshold = Integer.parseInt(thresholdConf); + + try { + multipleCyclesMapDetector = + detectorClass.getConstructor(String.class, Integer.class, + Integer.class, SingleCycleSingleDoubleAnomalyDetector.class).newInstance( + monitorItemName, cycles, threshold, + singleDoubleDetector); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + } + + } + else if (monitorDataSubTypeName.equals("StringMap")) { + + } + else { + throw new RuntimeException( + "Unknown Multiple Map Anomaly Detector Sub-Type"); + } + + return multipleCyclesMapDetector; + } + + @SuppressWarnings("unchecked") + private static MultipleCyclesAnomalyDetector generateMultipleCyclesAnomalyDetector( + String monitorItemName, + String monitorDataTypeName, + String monitorDataSubTypeName, + Element strategy) { + + MultipleCyclesAnomalyDetector multipleCyclesDetector = null; + + if (monitorDataSubTypeName.endsWith("Map") == true) { + multipleCyclesDetector = + generateMultipleCyclesMapTypeAnomalyDetector(monitorItemName, + monitorDataTypeName, monitorDataSubTypeName, strategy); + } + else if (monitorDataSubTypeName.equals("Compound")) { + + String cyclesConf = null; + String detectorConf = null; + + NodeList subFieldNodes = strategy.getChildNodes(); + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element) subFieldNode; + if ("cycles".equals(subField.getTagName()) && subField.hasChildNodes()) { + cyclesConf = ((Text)subField.getFirstChild()).getData().trim(); + } + if ("detector".equals(subField.getTagName()) && subField.hasChildNodes()) { + detectorConf = ((Text)subField.getFirstChild()).getData().trim(); + } + } + + if (cyclesConf != null && detectorConf != null) { + + Class detectorClass = null; + try { + detectorClass = + (Class) Class.forName( + detectorConf); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + + if (detectorClass != null) { + + try { + multipleCyclesDetector = + detectorClass.getConstructor( + String.class, int.class).newInstance( + monitorItemName, new Integer(cyclesConf).intValue()); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + } else { + System.out.println("!!"); + } + } + + } + else { + throw new RuntimeException("Unknown Multiple Cycles Anomaly Detector Type"); + } + + return multipleCyclesDetector; + } + + public static AnomalyDetector generate( + String monitorItemName, + String monitorDataTypeName, + String monitorDataSubTypeName, + Element strategy) { + + AnomalyDetector detector = null; + + if (monitorDataTypeName.equals("SingleCycle")) { + detector = generateSingleCycleAnomalyDetector(monitorItemName, + monitorDataTypeName, monitorDataSubTypeName, strategy); + } + else if (monitorDataTypeName.equals("DoubleCycles")) { + detector = generateDoubleCyclesAnomalyDetector(monitorItemName, + monitorDataTypeName, monitorDataSubTypeName, strategy); + } + else if (monitorDataTypeName.equals("MultipleCycles")) { + detector = generateMultipleCyclesAnomalyDetector(monitorItemName, + monitorDataTypeName, monitorDataSubTypeName, strategy); + } + else { + throw new RuntimeException("Unknown Anomaly Detector Type"); + } + + return detector; + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesAnomalyDetector.java new file mode 100644 index 0000000..1e7cf00 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesAnomalyDetector.java @@ -0,0 +1,38 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class DoubleCyclesAnomalyDetector implements AnomalyDetector { + + private String monitorItemName = null; + + protected static final int DOUBLE_CYCLES_CONSTANT = 2; + + /** + * @param monitorItemName + */ + public DoubleCyclesAnomalyDetector(String monitorItemName) { + this.setMonitorItemName(monitorItemName); + } + + /** + * @param monitorItemName the monitorItemName to set + */ + public void setMonitorItemName(String monitorItemName) { + this.monitorItemName = monitorItemName; + } + + /** + * @return the monitorItemName + */ + public String getMonitorItemName() { + return monitorItemName; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesCompoundTypeAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesCompoundTypeAnomalyDetector.java new file mode 100644 index 0000000..81d4076 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesCompoundTypeAnomalyDetector.java @@ -0,0 +1,56 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.MonitorStrategy; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyWorker; + +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public abstract class DoubleCyclesCompoundTypeAnomalyDetector extends + DoubleCyclesAnomalyDetector { + + /** + * @param monitorItemName + */ + public DoubleCyclesCompoundTypeAnomalyDetector(String monitorItemName) { + super(monitorItemName); + } + + abstract protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyDataCollection( + NavigableMap multipleCyclesDataCollection); + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.AnomalyDetector#detect(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public AnomalyDetectionResult detect(MonitorStrategyWorker worker) { + + MonitorStrategy strategy = worker.getStrategy(); + NavigableMap dataProcessingCentre = + strategy.getDataProcessingCentre(); + + synchronized (dataProcessingCentre) { + Date acquisitionTime = worker.getDataAcquisitionTimepoint(); + return this.visitDoubleCyclesMonitorStrategyDataCollection( + strategy.retrieveMultipleCyclesStrategyDataCollection( + acquisitionTime, DOUBLE_CYCLES_CONSTANT)); + } + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleAnomalyDetector.java new file mode 100644 index 0000000..f57cb32 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleAnomalyDetector.java @@ -0,0 +1,40 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class DoubleCyclesSingleDoubleAnomalyDetector extends + DoubleCyclesSingleTypeAnomalyDetector { + + private double threshold = 0.0; + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleAnomalyDetector(String monitorItemName, + double threshold) { + super(monitorItemName); + this.setThreshold(threshold); + } + + /** + * @param threshold the threshold to set + */ + public void setThreshold(double threshold) { + this.threshold = threshold; + } + + /** + * @return the threshold + */ + public double getThreshold() { + return threshold; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateEqualToAnomalyDetector.java new file mode 100644 index 0000000..6f60e99 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateEqualToAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleDoubleRateEqualToAnomalyDetector extends + DoubleCyclesSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateEqualToAnomalyDetector( + String monitorItemName, double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateEqualToAnomalyDetector( + String monitorItemName, Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Double data1 = doubleCyclesData.get(date1).getData(); + Double data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) == this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " == " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateGreaterThanAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateGreaterThanAnomalyDetector.java new file mode 100644 index 0000000..05f00f8 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateGreaterThanAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleDoubleRateGreaterThanAnomalyDetector extends + DoubleCyclesSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateGreaterThanAnomalyDetector( + String monitorItemName, double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateGreaterThanAnomalyDetector( + String monitorItemName, Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Double data1 = doubleCyclesData.get(date1).getData(); + Double data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) > this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " > " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateGreaterThanOrEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateGreaterThanOrEqualToAnomalyDetector.java new file mode 100644 index 0000000..057f431 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateGreaterThanOrEqualToAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleDoubleRateGreaterThanOrEqualToAnomalyDetector + extends DoubleCyclesSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateGreaterThanOrEqualToAnomalyDetector( + String monitorItemName, double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateGreaterThanOrEqualToAnomalyDetector( + String monitorItemName, Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Double data1 = doubleCyclesData.get(date1).getData(); + Double data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) >= this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " >= " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateLessThanAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateLessThanAnomalyDetector.java new file mode 100644 index 0000000..0bad68f --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateLessThanAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleDoubleRateLessThanAnomalyDetector extends + DoubleCyclesSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateLessThanAnomalyDetector( + String monitorItemName, double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateLessThanAnomalyDetector( + String monitorItemName, Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Double data1 = doubleCyclesData.get(date1).getData(); + Double data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) < this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " < " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateLessThanOrEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateLessThanOrEqualToAnomalyDetector.java new file mode 100644 index 0000000..4985abd --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateLessThanOrEqualToAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleDoubleRateLessThanOrEqualToAnomalyDetector + extends DoubleCyclesSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateLessThanOrEqualToAnomalyDetector( + String monitorItemName, double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateLessThanOrEqualToAnomalyDetector( + String monitorItemName, Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Double data1 = doubleCyclesData.get(date1).getData(); + Double data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) <= this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " <= " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateNotEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateNotEqualToAnomalyDetector.java new file mode 100644 index 0000000..e9a0d5b --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleDoubleRateNotEqualToAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleDoubleRateNotEqualToAnomalyDetector extends + DoubleCyclesSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateNotEqualToAnomalyDetector( + String monitorItemName, double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleDoubleRateNotEqualToAnomalyDetector( + String monitorItemName, Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Double data1 = doubleCyclesData.get(date1).getData(); + Double data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) != this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " != " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongAnomalyDetector.java new file mode 100644 index 0000000..6120d62 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongAnomalyDetector.java @@ -0,0 +1,40 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class DoubleCyclesSingleLongAnomalyDetector extends + DoubleCyclesSingleTypeAnomalyDetector { + + private long threshold = 0; + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongAnomalyDetector(String monitorItemName, + long threshold) { + super(monitorItemName); + this.setThreshold(threshold); + } + + /** + * @param threshold the threshold to set + */ + public void setThreshold(long threshold) { + this.threshold = threshold; + } + + /** + * @return the threshold + */ + public long getThreshold() { + return threshold; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateEqualToAnomalyDetector.java new file mode 100644 index 0000000..fe94d90 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateEqualToAnomalyDetector.java @@ -0,0 +1,82 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleLongRateEqualToAnomalyDetector extends + DoubleCyclesSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateEqualToAnomalyDetector(String monitorItemName, + long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateEqualToAnomalyDetector(String monitorItemName, + Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Long data1 = doubleCyclesData.get(date1).getData(); + Long data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) == this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " == " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateGreaterThanAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateGreaterThanAnomalyDetector.java new file mode 100644 index 0000000..b6ef718 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateGreaterThanAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleLongRateGreaterThanAnomalyDetector extends + DoubleCyclesSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateGreaterThanAnomalyDetector( + String monitorItemName, long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateGreaterThanAnomalyDetector( + String monitorItemName, Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Long data1 = doubleCyclesData.get(date1).getData(); + Long data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) > this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " > " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateGreaterThanOrEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateGreaterThanOrEqualToAnomalyDetector.java new file mode 100644 index 0000000..0d0a7c2 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateGreaterThanOrEqualToAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleLongRateGreaterThanOrEqualToAnomalyDetector extends + DoubleCyclesSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateGreaterThanOrEqualToAnomalyDetector( + String monitorItemName, long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateGreaterThanOrEqualToAnomalyDetector( + String monitorItemName, Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Long data1 = doubleCyclesData.get(date1).getData(); + Long data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) >= this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " >= " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateLessThanAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateLessThanAnomalyDetector.java new file mode 100644 index 0000000..4d996bc --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateLessThanAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleLongRateLessThanAnomalyDetector extends + DoubleCyclesSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateLessThanAnomalyDetector( + String monitorItemName, long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateLessThanAnomalyDetector( + String monitorItemName, Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Long data1 = doubleCyclesData.get(date1).getData(); + Long data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) < this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " < " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateLessThanOrEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateLessThanOrEqualToAnomalyDetector.java new file mode 100644 index 0000000..8057c20 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateLessThanOrEqualToAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleLongRateLessThanOrEqualToAnomalyDetector extends + DoubleCyclesSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateLessThanOrEqualToAnomalyDetector( + String monitorItemName, long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateLessThanOrEqualToAnomalyDetector( + String monitorItemName, Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Long data1 = doubleCyclesData.get(date1).getData(); + Long data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) <= this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " <= " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateNotEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateNotEqualToAnomalyDetector.java new file mode 100644 index 0000000..ad9525d --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleLongRateNotEqualToAnomalyDetector.java @@ -0,0 +1,81 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class DoubleCyclesSingleLongRateNotEqualToAnomalyDetector extends + DoubleCyclesSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateNotEqualToAnomalyDetector( + String monitorItemName, long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public DoubleCyclesSingleLongRateNotEqualToAnomalyDetector( + String monitorItemName, Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.DoubleCyclesSingleTypeAnomalyDetector#visitDoubleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData) { + + if (doubleCyclesData.size() < DOUBLE_CYCLES_CONSTANT) { + return null; + } + + Date date1 = doubleCyclesData.firstKey(); + Date date2 = doubleCyclesData.lastKey(); + Long time = date2.getTime() - date1.getTime(); + + Long data1 = doubleCyclesData.get(date1).getData(); + Long data2 = doubleCyclesData.get(date2).getData(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String detectionInfo = + "[" + dateFormat.format(date2) + "] " + + getMonitorItemName() + " = " + data2 + + " -> " + + "[" + dateFormat.format(date1) + "] " + + getMonitorItemName() + " = " + data1; + + boolean anomalous = (((data2 - data1) / time) != this.getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " != " + getThreshold() + "\n" + detectionInfo; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleTypeAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleTypeAnomalyDetector.java new file mode 100644 index 0000000..f1be709 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/DoubleCyclesSingleTypeAnomalyDetector.java @@ -0,0 +1,52 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.util.Date; +import java.util.NavigableMap; + +import hma.monitor.MonitorStrategy; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyWorker; + +/** + * @author guoyezhi + * + */ +public abstract class DoubleCyclesSingleTypeAnomalyDetector extends + DoubleCyclesAnomalyDetector { + + /** + * @param monitorItemName + */ + public DoubleCyclesSingleTypeAnomalyDetector(String monitorItemName) { + super(monitorItemName); + } + + abstract protected AnomalyDetectionResult visitDoubleCyclesMonitorStrategyData( + NavigableMap> doubleCyclesData); + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.AnomalyDetector#detect(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public AnomalyDetectionResult detect(MonitorStrategyWorker worker) { + + MonitorStrategy strategy = worker.getStrategy(); + NavigableMap dataProcessingCentre = + strategy.getDataProcessingCentre(); + + synchronized (dataProcessingCentre) { + Date acquisitionTime = worker.getDataAcquisitionTimepoint(); + return this.visitDoubleCyclesMonitorStrategyData( + strategy.retrieveMultipleCyclesStrategyData( + this.getMonitorItemName(), + acquisitionTime, + DOUBLE_CYCLES_CONSTANT)); + } + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/MultipleCyclesAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/MultipleCyclesAnomalyDetector.java new file mode 100644 index 0000000..058e9d4 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/MultipleCyclesAnomalyDetector.java @@ -0,0 +1,55 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class MultipleCyclesAnomalyDetector implements AnomalyDetector { + + private String monitorItemName = null; + + private int cycles = 0; + + /** + * @param monitorItemName + * @param cycles + */ + public MultipleCyclesAnomalyDetector(String monitorItemName, + int cycles) { + this.setMonitorItemName(monitorItemName); + this.setCycles(cycles); + } + + /** + * @param monitorItemName the monitorItemName to set + */ + public void setMonitorItemName(String monitorItemName) { + this.monitorItemName = monitorItemName; + } + + /** + * @return the monitorItemName + */ + public String getMonitorItemName() { + return monitorItemName; + } + + /** + * @param cycles the cycles to set + */ + public void setCycles(int cycles) { + this.cycles = cycles; + } + + /** + * @return the cycles + */ + public int getCycles() { + return cycles; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/MultipleCyclesCompoundTypeAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/MultipleCyclesCompoundTypeAnomalyDetector.java new file mode 100644 index 0000000..9803b11 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/MultipleCyclesCompoundTypeAnomalyDetector.java @@ -0,0 +1,50 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.MonitorStrategy; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyWorker; + +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public abstract class MultipleCyclesCompoundTypeAnomalyDetector extends + MultipleCyclesAnomalyDetector { + + /** + * @param monitorItemName + * @param cycles + */ + public MultipleCyclesCompoundTypeAnomalyDetector(String monitorItemName, + int cycles) { + super(monitorItemName, cycles); + } + + abstract protected AnomalyDetectionResult visitMultipleCyclesMonitorStrategyDataCollection( + NavigableMap multipleCyclesDataCollection); + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.AnomalyDetector#detect(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public AnomalyDetectionResult detect(MonitorStrategyWorker worker) { + + MonitorStrategy strategy = worker.getStrategy(); + NavigableMap dataProcessingCentre = + strategy.getDataProcessingCentre(); + + synchronized (dataProcessingCentre) { + Date acquisitionTime = worker.getDataAcquisitionTimepoint(); + return this.visitMultipleCyclesMonitorStrategyDataCollection( + strategy.retrieveMultipleCyclesStrategyDataCollection( + acquisitionTime, this.getCycles())); + } + } + +} diff --git a/src/java/hma/monitor/strategy/detect/MultipleCyclesDoubleMapAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/MultipleCyclesDoubleMapAnomalyDetector.java new file mode 100644 index 0000000..ef50b48 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/MultipleCyclesDoubleMapAnomalyDetector.java @@ -0,0 +1,40 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class MultipleCyclesDoubleMapAnomalyDetector extends + MultipleCyclesMapTypeAnomalyDetector { + + private SingleCycleSingleDoubleAnomalyDetector singleCycleDetector = null; + + /** + * @param monitorItemName + * @param cycles + */ + public MultipleCyclesDoubleMapAnomalyDetector(String monitorItemName, + int cycles, SingleCycleSingleDoubleAnomalyDetector singleDetector) { + super(monitorItemName, cycles); + this.setSingleDetector(singleDetector); + } + + /** + * @param singleCycleDetector the singleCycleDetector to set + */ + public void setSingleDetector(SingleCycleSingleDoubleAnomalyDetector singleDetector) { + this.singleCycleDetector = singleDetector; + } + + /** + * @return the singleCycleDetector + */ + public SingleCycleSingleDoubleAnomalyDetector getSingleDetector() { + return singleCycleDetector; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/MultipleCyclesDoubleMapEventContinuityAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/MultipleCyclesDoubleMapEventContinuityAnomalyDetector.java new file mode 100644 index 0000000..6c9121d --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/MultipleCyclesDoubleMapEventContinuityAnomalyDetector.java @@ -0,0 +1,146 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; + +/** + * @author guoyezhi + * + */ +public class MultipleCyclesDoubleMapEventContinuityAnomalyDetector extends + MultipleCyclesDoubleMapAnomalyDetector { + + private static final SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param cycles + * @param threshold + * @param singleCycleDetector + */ + public MultipleCyclesDoubleMapEventContinuityAnomalyDetector( + String monitorItemName, int cycles, int threshold, + SingleCycleSingleDoubleAnomalyDetector singleCycleDetector) { + super(monitorItemName, cycles, singleCycleDetector); + // nothing needed to do with @param threshold + } + + /** + * @param monitorItemName + * @param cycles + * @param threshold + * @param singleCycleDetector + */ + public MultipleCyclesDoubleMapEventContinuityAnomalyDetector( + String monitorItemName, Integer cycles, Integer threshold, + SingleCycleSingleDoubleAnomalyDetector singleCycleDetector) { + super(monitorItemName, cycles.intValue(), singleCycleDetector); + // nothing needed to do with @param threshold + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.MultipleCyclesMapTypeAnomalyDetector#visitMultipleCyclesMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitMultipleCyclesMonitorStrategyData( + NavigableMap>> multipleCyclesData) { + + if (multipleCyclesData.size() < this.getCycles()) { + return null; + } + + StringBuilder detectionInfoBuilder = new StringBuilder(); + detectionInfoBuilder.append( + "[" + dateFormat.format(multipleCyclesData.lastKey()) + "] " + + "The last cycle detection result:\n"); + Iterator> iter2 = + multipleCyclesData.lastEntry().getValue().getData().entrySet().iterator(); + while (iter2.hasNext()) { + Map.Entry entry = iter2.next(); + String target = entry.getKey(); + Double value = entry.getValue(); + detectionInfoBuilder.append("\t" + target + " -> " + value + "\n"); + } + String detectionInfo = detectionInfoBuilder.toString(); + + + Date firstKey = multipleCyclesData.firstKey(); + int i = this.getCycles(); + Date d = firstKey; + Set set = + multipleCyclesData.firstEntry().getValue().getData().keySet(); + + Map toDetect = null; + Map detected = null; + + SingleCycleSingleDoubleAnomalyDetector singleCycleDetector = + this.getSingleDetector(); + + while (d != null && i > 0 && (set.size() != 0)) { + + toDetect = multipleCyclesData.get(d).getData(); + detected = new HashMap(); + + Iterator iter = set.iterator(); + + while (iter.hasNext()) { + String key = iter.next(); + Double value = toDetect.get(key); + if (value == null) continue; + MonitorStrategyData wrapper = + new MonitorStrategyData(null, value, i); + AnomalyDetectionResult res = + singleCycleDetector.visitMonitorStrategyData(wrapper); + if (res.isAnomalous() == true) { + detected.put(key, value); + } + } + + d = multipleCyclesData.higherKey(d); + i--; + set = detected.keySet(); + } + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + String alarmInfo = null; + StringBuilder alarmInfoBuilder = new StringBuilder(); + + if (i == 0 && (set.size() > 0)) { + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + + alarmInfoBuilder.append( + "Total Anomalous Target Num: " + set.size() + "\n"); + alarmInfoBuilder.append("Anomalous Target Status of last cycle:\n"); + Iterator> iter = + detected.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String target = entry.getKey(); + Double value = entry.getValue(); + alarmInfoBuilder.append("\t" + target + " -> " + value + "\n"); + } + alarmInfo = alarmInfoBuilder.toString(); + + return new AnomalyDetectionResult( + true, alarmLevel, alarmInfo, detectionInfo); + } + + return new AnomalyDetectionResult( + false, alarmLevel, alarmInfo, detectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/MultipleCyclesLongMapAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/MultipleCyclesLongMapAnomalyDetector.java new file mode 100644 index 0000000..0b6e6c3 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/MultipleCyclesLongMapAnomalyDetector.java @@ -0,0 +1,40 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class MultipleCyclesLongMapAnomalyDetector extends + MultipleCyclesMapTypeAnomalyDetector { + + private SingleCycleSingleLongAnomalyDetector singleCycleDetector = null; + + /** + * @param monitorItemName + * @param cycles + */ + public MultipleCyclesLongMapAnomalyDetector(String monitorItemName, + int cycles, SingleCycleSingleLongAnomalyDetector singleDetector) { + super(monitorItemName, cycles); + this.setSingleDetector(singleDetector); + } + + /** + * @param singleCycleDetector the singleCycleDetector to set + */ + public void setSingleDetector(SingleCycleSingleLongAnomalyDetector singleDetector) { + this.singleCycleDetector = singleDetector; + } + + /** + * @return the singleCycleDetector + */ + public SingleCycleSingleLongAnomalyDetector getSingleDetector() { + return singleCycleDetector; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/MultipleCyclesLongMapEventContinuityAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/MultipleCyclesLongMapEventContinuityAnomalyDetector.java new file mode 100644 index 0000000..0eeae1c --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/MultipleCyclesLongMapEventContinuityAnomalyDetector.java @@ -0,0 +1,144 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; + +/** + * @author guoyezhi + * + */ +public class MultipleCyclesLongMapEventContinuityAnomalyDetector extends + MultipleCyclesLongMapAnomalyDetector { + + private static final SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param cycles + * @param singleCycleDetector + */ + public MultipleCyclesLongMapEventContinuityAnomalyDetector( + String monitorItemName, int cycles, int threshold, + SingleCycleSingleLongAnomalyDetector singleCycleDetector) { + super(monitorItemName, cycles, singleCycleDetector); + // nothing needed to do with @param threshold + } + + /** + * @param monitorItemName + * @param cycles + * @param threshold + * @param singleCycleDetector + */ + public MultipleCyclesLongMapEventContinuityAnomalyDetector( + String monitorItemName, Integer cycles, Integer threshold, + SingleCycleSingleLongAnomalyDetector singleCycleDetector) { + super(monitorItemName, cycles.intValue(), singleCycleDetector); + // nothing needed to do with @param threshold + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.MultipleCyclesAnomalyDetector#visitMultipleMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitMultipleCyclesMonitorStrategyData( + NavigableMap>> multipleCyclesData) { + + if (multipleCyclesData.size() < this.getCycles()) + return null; + + StringBuilder fullInfoBuilder = new StringBuilder(); + fullInfoBuilder.append( + "[" + dateFormat.format(multipleCyclesData.lastKey()) + "] " + + "The last cycle detection result:\n"); + Iterator> iter2 = + multipleCyclesData.lastEntry().getValue().getData().entrySet().iterator(); + while (iter2.hasNext()) { + Map.Entry entry = iter2.next(); + String target = entry.getKey(); + Long value = entry.getValue(); + fullInfoBuilder.append("\t" + target + " -> " + value + "\n"); + } + String fullResultInfo = fullInfoBuilder.toString(); + + + Date firstKey = multipleCyclesData.firstKey(); + int i = this.getCycles(); + Date d = firstKey; + Set set = + multipleCyclesData.lastEntry().getValue().getData().keySet(); + + Map toDetect = null; + Map detected = null; + + SingleCycleSingleLongAnomalyDetector singleCycleDetector = + this.getSingleDetector(); + + while (d != null && i > 0 && (set.size() != 0)) { + + toDetect = multipleCyclesData.get(d).getData(); + detected = new HashMap(); + + Iterator iter = set.iterator(); + + while (iter.hasNext()) { + String key = iter.next(); + Long value = toDetect.get(key); + if (value == null) continue; + MonitorStrategyData wrapper = + new MonitorStrategyData(null, value, i); + AnomalyDetectionResult res = + singleCycleDetector.visitMonitorStrategyData(wrapper); + if (res.isAnomalous() == true) { + detected.put(key, value); + } + } + + d = multipleCyclesData.higherKey(d); + i--; + set = detected.keySet(); + } + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + String keyResultInfo = null; + StringBuilder keyInfoBuilder = new StringBuilder(); + + if (i == 0 && (set.size() > 0)) { + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + + keyInfoBuilder.append( + "Total Anomalous Target Num: " + set.size() + "\n"); + keyInfoBuilder.append("Anomalous Target Status of last cycle:\n"); + Iterator> iter = + detected.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String target = entry.getKey(); + Long value = entry.getValue(); + keyInfoBuilder.append("\t" + target + " -> " + value + "\n"); + } + keyResultInfo = keyInfoBuilder.toString(); + + return new AnomalyDetectionResult( + true, alarmLevel, keyResultInfo, fullResultInfo); + } + + return new AnomalyDetectionResult( + false, alarmLevel, keyResultInfo, fullResultInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/MultipleCyclesLongMapEventProbabilityAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/MultipleCyclesLongMapEventProbabilityAnomalyDetector.java new file mode 100644 index 0000000..4445b00 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/MultipleCyclesLongMapEventProbabilityAnomalyDetector.java @@ -0,0 +1,151 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class MultipleCyclesLongMapEventProbabilityAnomalyDetector extends + MultipleCyclesLongMapAnomalyDetector { + + private int occurenceThreshold = 0; + + private static final SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param cycles + * @param occurenceThreshold + * @param singleCycleDetector + */ + public MultipleCyclesLongMapEventProbabilityAnomalyDetector( + String monitorItemName, int cycles, int threshold, + SingleCycleSingleLongAnomalyDetector singleCycleDetector) { + super(monitorItemName, cycles, singleCycleDetector); + this.occurenceThreshold = threshold; + } + + public MultipleCyclesLongMapEventProbabilityAnomalyDetector( + String monitorItemName, Integer cycles, Integer threshold, + SingleCycleSingleLongAnomalyDetector singleCycleDetector) { + this(monitorItemName, cycles.intValue(), threshold.intValue(), + singleCycleDetector); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.MultipleCyclesAnomalyDetector#visitMultipleMonitorStrategyData(java.util.NavigableMap) + */ + @Override + protected AnomalyDetectionResult visitMultipleCyclesMonitorStrategyData( + NavigableMap>> multipleCyclesData) { + + if (multipleCyclesData.size() < this.getCycles()) + return null; + + StringBuilder fullInfoBuilder = new StringBuilder(); + fullInfoBuilder.append( + "[" + dateFormat.format(multipleCyclesData.lastKey()) + "] " + + "The last cycle detection result:\n"); + Iterator> it = + multipleCyclesData.lastEntry().getValue().getData().entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + String target = entry.getKey(); + Long value = entry.getValue(); + fullInfoBuilder.append("\t" + target + " -> " + value + "\n"); + } + String fullResultInfo = fullInfoBuilder.toString(); + + + int i = this.getCycles(); + Date d = multipleCyclesData.lastKey(); + Map toDetect = null; + SingleCycleSingleLongAnomalyDetector singleCycleDetector = + this.getSingleDetector(); + Map detectionVar = new HashMap(); + + while (d != null && i > 0) { + + toDetect = multipleCyclesData.get(d).getData(); + + Iterator> iter = toDetect.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String key = entry.getKey(); + Long value = entry.getValue(); + if (value == null) continue; + MonitorStrategyData wrapper = + new MonitorStrategyData(null, value, i); + AnomalyDetectionResult res = + singleCycleDetector.visitMonitorStrategyData(wrapper); + if (res.isAnomalous() == true) { + Integer occurences = detectionVar.get(key); + if (occurences == null) { + detectionVar.put(key, 1); + } else { + detectionVar.put(key, occurences + 1); + } + } + } + + d = multipleCyclesData.lowerKey(d); + i--; + } + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + String keyResultInfo = null; + StringBuilder keyInfoBuilder = new StringBuilder(); + Map detectionResult = new HashMap(); + + if (i == 0 && (detectionVar.size() > 0)) { + Iterator> iter = + detectionVar.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String key = entry.getKey(); + Integer value = entry.getValue(); + if (value >= this.occurenceThreshold) { + detectionResult.put(key, value); + } + } + if (detectionResult.size() > 0) { + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + + keyInfoBuilder.append("Total Anomalous Target Num: " + + detectionResult.size() + "\n"); + keyInfoBuilder.append("Anomalous Target Status of last cycle:\n"); + Iterator> iter2 = detectionResult.entrySet().iterator(); + while (iter2.hasNext()) { + Map.Entry entry = iter2.next(); + String target = entry.getKey(); + Integer value = entry.getValue(); + keyInfoBuilder.append("\t" + target + " : " + + value + " / " + this.getCycles() + "\n"); + } + keyResultInfo = keyInfoBuilder.toString(); + + return new AnomalyDetectionResult( + true, alarmLevel, keyResultInfo, fullResultInfo); + } + } + + return new AnomalyDetectionResult( + false, alarmLevel, keyResultInfo, fullResultInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/MultipleCyclesMapTypeAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/MultipleCyclesMapTypeAnomalyDetector.java new file mode 100644 index 0000000..af0410d --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/MultipleCyclesMapTypeAnomalyDetector.java @@ -0,0 +1,51 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.MonitorStrategy; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyWorker; + +import java.util.Date; +import java.util.Map; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public abstract class MultipleCyclesMapTypeAnomalyDetector extends + MultipleCyclesAnomalyDetector { + + public MultipleCyclesMapTypeAnomalyDetector(String monitorItemName, + int cycles) { + super(monitorItemName, cycles); + } + + abstract protected AnomalyDetectionResult visitMultipleCyclesMonitorStrategyData( + NavigableMap>> multipleCyclesData); + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.AnomalyDetector#detect(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public AnomalyDetectionResult detect(MonitorStrategyWorker worker) { + + MonitorStrategy strategy = worker.getStrategy(); + NavigableMap dataProcessingCentre = + strategy.getDataProcessingCentre(); + + synchronized (dataProcessingCentre) { + Date acquisitionTime = worker.getDataAcquisitionTimepoint(); + return this.visitMultipleCyclesMonitorStrategyData( + strategy.>retrieveMultipleCyclesStrategyData( + getMonitorItemName(), + acquisitionTime, + getCycles())); + } + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleAnomalyDetector.java new file mode 100644 index 0000000..785ef29 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleAnomalyDetector.java @@ -0,0 +1,36 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class SingleCycleAnomalyDetector implements AnomalyDetector { + + private String monitorItemName = null; + + /** + * @param monitorItemName + */ + public SingleCycleAnomalyDetector(String monitorItemName) { + this.setMonitorItemName(monitorItemName); + } + + /** + * @param monitorItemName the monitorItemName to set + */ + public void setMonitorItemName(String monitorItemName) { + this.monitorItemName = monitorItemName; + } + + /** + * @return the monitorItemName + */ + public String getMonitorItemName() { + return monitorItemName; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleCompoundTypeAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleCompoundTypeAnomalyDetector.java new file mode 100644 index 0000000..1cd11d8 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleCompoundTypeAnomalyDetector.java @@ -0,0 +1,48 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import hma.monitor.MonitorStrategy; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyWorker; + +import java.util.Date; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public abstract class SingleCycleCompoundTypeAnomalyDetector extends + SingleCycleAnomalyDetector { + + /** + * @param monitorItemName + */ + public SingleCycleCompoundTypeAnomalyDetector(String monitorItemName) { + super(monitorItemName); + } + + abstract protected AnomalyDetectionResult visitMonitorStrategyDataCollection( + MonitorStrategyDataCollection strategyDataCollection); + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.AnomalyDetector#detect(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public AnomalyDetectionResult detect(MonitorStrategyWorker worker) { + + MonitorStrategy strategy = worker.getStrategy(); + NavigableMap dataProcessingCentre = + strategy.getDataProcessingCentre(); + + synchronized (dataProcessingCentre) { + Date acquisitionTime = worker.getDataAcquisitionTimepoint(); + return this.visitMonitorStrategyDataCollection( + strategy.retrieveSingleCycleStrategyDataCollection(acquisitionTime)); + } + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleDoubleMapAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleDoubleMapAnomalyDetector.java new file mode 100644 index 0000000..84c1e22 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleDoubleMapAnomalyDetector.java @@ -0,0 +1,39 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class SingleCycleDoubleMapAnomalyDetector extends + SingleCycleMapTypeAnomalyDetector { + + private SingleCycleSingleDoubleAnomalyDetector singleTypeDetector = null; + + /** + * @param monitorItemName + */ + public SingleCycleDoubleMapAnomalyDetector(String monitorItemName, + SingleCycleSingleDoubleAnomalyDetector singleTypeDetector) { + super(monitorItemName); + this.setSingleTypeDetector(singleTypeDetector); + } + + /** + * @param singleTypeDetector the singleTypeDetector to set + */ + public void setSingleTypeDetector(SingleCycleSingleDoubleAnomalyDetector singleTypeDetector) { + this.singleTypeDetector = singleTypeDetector; + } + + /** + * @return the singleTypeDetector + */ + public SingleCycleSingleDoubleAnomalyDetector getSingleTypeDetector() { + return singleTypeDetector; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleLongMapAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleLongMapAnomalyDetector.java new file mode 100644 index 0000000..a8bf10d --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleLongMapAnomalyDetector.java @@ -0,0 +1,39 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class SingleCycleLongMapAnomalyDetector extends + SingleCycleMapTypeAnomalyDetector { + + private SingleCycleSingleLongAnomalyDetector singleTypeDetector = null; + + /** + * @param monitorItemName + */ + public SingleCycleLongMapAnomalyDetector(String monitorItemName, + SingleCycleSingleLongAnomalyDetector singleTypeDetector) { + super(monitorItemName); + this.setSingleTypeDetector(singleTypeDetector); + } + + /** + * @param singleTypeDetector the singleTypeDetector to set + */ + public void setSingleTypeDetector(SingleCycleSingleLongAnomalyDetector singleTypeDetector) { + this.singleTypeDetector = singleTypeDetector; + } + + /** + * @return the singleTypeDetector + */ + public SingleCycleSingleLongAnomalyDetector getSingleTypeDetector() { + return singleTypeDetector; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleMapTypeAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleMapTypeAnomalyDetector.java new file mode 100644 index 0000000..b727ceb --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleMapTypeAnomalyDetector.java @@ -0,0 +1,51 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.util.Date; +import java.util.Map; +import java.util.NavigableMap; + +import hma.monitor.MonitorStrategy; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyWorker; + +/** + * @author guoyezhi + * + */ +public abstract class SingleCycleMapTypeAnomalyDetector extends + SingleCycleAnomalyDetector { + + /** + * @param monitorItemName + */ + public SingleCycleMapTypeAnomalyDetector(String monitorItemName) { + super(monitorItemName); + } + + abstract protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData> strategyData); + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.AnomalyDetector#detect(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public AnomalyDetectionResult detect(MonitorStrategyWorker worker) { + + MonitorStrategy strategy = worker.getStrategy(); + NavigableMap dataProcessingCentre = + strategy.getDataProcessingCentre(); + + synchronized (dataProcessingCentre) { + Date acquisitionTime = worker.getDataAcquisitionTimepoint(); + return this.visitMonitorStrategyData( + strategy.>retrieveSingleCycleStrategyData( + getMonitorItemName(), acquisitionTime)); + } + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleAnomalyDetector.java new file mode 100644 index 0000000..a03a98c --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleAnomalyDetector.java @@ -0,0 +1,41 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class SingleCycleSingleDoubleAnomalyDetector extends + SingleCycleSingleTypeAnomalyDetector { + + private double threshold = 0.0; + + /** + * + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleAnomalyDetector(String monitorItemName, + double threshold) { + super(monitorItemName); + this.setThreshold(threshold); + } + + /** + * @param threshold the threshold to set + */ + public void setThreshold(double threshold) { + this.threshold = threshold; + } + + /** + * @return the threshold + */ + public double getThreshold() { + return threshold; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleEqualToAnomalyDetector.java new file mode 100644 index 0000000..1120001 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleEqualToAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleDoubleEqualToAnomalyDetector extends + SingleCycleSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleEqualToAnomalyDetector(String monitorItemName, + double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleEqualToAnomalyDetector(String monitorItemName, + Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + double data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data == getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " == " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleGreaterThanAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleGreaterThanAnomalyDetector.java new file mode 100644 index 0000000..7357127 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleGreaterThanAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleDoubleGreaterThanAnomalyDetector extends + SingleCycleSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleGreaterThanAnomalyDetector(String monitorItemName, + double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleGreaterThanAnomalyDetector(String monitorItemName, + Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + double data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data > getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " > " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleGreaterThanOrEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleGreaterThanOrEqualToAnomalyDetector.java new file mode 100644 index 0000000..4c2605c --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleGreaterThanOrEqualToAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleDoubleGreaterThanOrEqualToAnomalyDetector extends + SingleCycleSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleGreaterThanOrEqualToAnomalyDetector( + String monitorItemName, double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleGreaterThanOrEqualToAnomalyDetector( + String monitorItemName, Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + double data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data >= getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " >= " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleLessThanAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleLessThanAnomalyDetector.java new file mode 100644 index 0000000..00ef523 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleLessThanAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleDoubleLessThanAnomalyDetector extends + SingleCycleSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleLessThanAnomalyDetector(String monitorItemName, + double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleLessThanAnomalyDetector(String monitorItemName, + Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + double data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data < getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " < " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleLessThanOrEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleLessThanOrEqualToAnomalyDetector.java new file mode 100644 index 0000000..af2ccb7 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleLessThanOrEqualToAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleDoubleLessThanOrEqualToAnomalyDetector extends + SingleCycleSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleLessThanOrEqualToAnomalyDetector(String monitorItemName, + double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleLessThanOrEqualToAnomalyDetector(String monitorItemName, + Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + double data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data <= getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " <= " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleNotEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleNotEqualToAnomalyDetector.java new file mode 100644 index 0000000..9dc7565 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleDoubleNotEqualToAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleDoubleNotEqualToAnomalyDetector extends + SingleCycleSingleDoubleAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleNotEqualToAnomalyDetector(String monitorItemName, + double threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleDoubleNotEqualToAnomalyDetector(String monitorItemName, + Double threshold) { + this(monitorItemName, threshold.doubleValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + double data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data != getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " != " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongAnomalyDetector.java new file mode 100644 index 0000000..577c56f --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongAnomalyDetector.java @@ -0,0 +1,40 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class SingleCycleSingleLongAnomalyDetector extends + SingleCycleSingleTypeAnomalyDetector { + + private long threshold = 0; + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongAnomalyDetector(String monitorItemName, + long threshold) { + super(monitorItemName); + this.setThreshold(threshold); + } + + /** + * @param threshold the threshold to set + */ + public void setThreshold(long threshold) { + this.threshold = threshold; + } + + /** + * @return the threshold + */ + public long getThreshold() { + return threshold; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongEqualToAnomalyDetector.java new file mode 100644 index 0000000..789d55c --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongEqualToAnomalyDetector.java @@ -0,0 +1,84 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleLongEqualToAnomalyDetector extends + SingleCycleSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongEqualToAnomalyDetector(String monitorItemName, + long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongEqualToAnomalyDetector(String monitorItemName, + Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data == getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " == " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + + public static void main(String[] args) { + SingleCycleSingleLongEqualToAnomalyDetector detector = + new SingleCycleSingleLongEqualToAnomalyDetector("FooItem", 777); + AnomalyDetectionResult res = detector.visitMonitorStrategyData( + new MonitorStrategyData("FooBar", 777L, 123456)); + System.out.println(res.isAnomalous()); + System.out.println( + MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue( + res.getAlarmLevel())); + System.out.println(res.getAlarmInfo()); + System.out.println(res.getDectionInfo()); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongGreaterThanAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongGreaterThanAnomalyDetector.java new file mode 100644 index 0000000..2e6f89b --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongGreaterThanAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleLongGreaterThanAnomalyDetector extends + SingleCycleSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongGreaterThanAnomalyDetector(String monitorItemName, + long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongGreaterThanAnomalyDetector(String monitorItemName, + Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data > getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " > " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongGreaterThanOrEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongGreaterThanOrEqualToAnomalyDetector.java new file mode 100644 index 0000000..8aad346 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongGreaterThanOrEqualToAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleLongGreaterThanOrEqualToAnomalyDetector extends + SingleCycleSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongGreaterThanOrEqualToAnomalyDetector( + String monitorItemName, long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongGreaterThanOrEqualToAnomalyDetector( + String monitorItemName, Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data >= getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " >= " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongLessThanAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongLessThanAnomalyDetector.java new file mode 100644 index 0000000..81398b2 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongLessThanAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleLongLessThanAnomalyDetector extends + SingleCycleSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongLessThanAnomalyDetector(String monitorItemName, + long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongLessThanAnomalyDetector(String monitorItemName, + Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data < getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " < " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongLessThanOrEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongLessThanOrEqualToAnomalyDetector.java new file mode 100644 index 0000000..c6e402e --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongLessThanOrEqualToAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleLongLessThanOrEqualToAnomalyDetector extends + SingleCycleSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongLessThanOrEqualToAnomalyDetector(String monitorItemName, + long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongLessThanOrEqualToAnomalyDetector(String monitorItemName, + Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data <= getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " <= " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongNotEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongNotEqualToAnomalyDetector.java new file mode 100644 index 0000000..304ad80 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleLongNotEqualToAnomalyDetector.java @@ -0,0 +1,71 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleLongNotEqualToAnomalyDetector extends + SingleCycleSingleLongAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongNotEqualToAnomalyDetector(String monitorItemName, + long threshold) { + super(monitorItemName, threshold); + } + + /** + * @param monitorItemName + * @param threshold + */ + public SingleCycleSingleLongNotEqualToAnomalyDetector(String monitorItemName, + Long threshold) { + this(monitorItemName, threshold.longValue()); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long data = strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + + boolean anomalous = (data != getThreshold()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " != " + getThreshold() + "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " = " + data; + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringAnomalyDetector.java new file mode 100644 index 0000000..24db28c --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringAnomalyDetector.java @@ -0,0 +1,40 @@ +/** + * + */ +package hma.monitor.strategy.detect; + + +/** + * @author guoyezhi + * + */ +public abstract class SingleCycleSingleStringAnomalyDetector extends + SingleCycleSingleTypeAnomalyDetector { + + private String pattern = null; + + /** + * @param monitorItemName + * @param pattern + */ + public SingleCycleSingleStringAnomalyDetector(String monitorItemName, + String pattern) { + super(monitorItemName); + this.setPattern(pattern); + } + + /** + * @param pattern the pattern to set + */ + public void setPattern(String pattern) { + this.pattern = pattern; + } + + /** + * @return the pattern + */ + public String getPattern() { + return pattern; + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringEqualToAnomalyDetector.java new file mode 100644 index 0000000..49fcd63 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringEqualToAnomalyDetector.java @@ -0,0 +1,62 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleStringEqualToAnomalyDetector extends + SingleCycleSingleStringAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param pattern + */ + public SingleCycleSingleStringEqualToAnomalyDetector(String monitorItemName, + String pattern) { + super(monitorItemName, pattern); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + + boolean anomalous = + (strategyData.getData().equals(this.getPattern())); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " EQUALS TO '" + getPattern() + "'\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringMatchRegExpAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringMatchRegExpAnomalyDetector.java new file mode 100644 index 0000000..c2ce733 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringMatchRegExpAnomalyDetector.java @@ -0,0 +1,63 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; +import java.util.regex.Pattern; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleStringMatchRegExpAnomalyDetector extends + SingleCycleSingleStringAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param pattern + */ + public SingleCycleSingleStringMatchRegExpAnomalyDetector(String monitorItemName, + String pattern) { + super(monitorItemName, pattern); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + + Pattern p = Pattern.compile(this.getPattern()); + boolean anomalous = p.matcher(strategyData.getData()).find(); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " MATCHES '" + getPattern() + "'\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringNotEmptyAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringNotEmptyAnomalyDetector.java new file mode 100644 index 0000000..d01e6fe --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringNotEmptyAnomalyDetector.java @@ -0,0 +1,62 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleStringNotEmptyAnomalyDetector extends + SingleCycleSingleStringAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param pattern + */ + public SingleCycleSingleStringNotEmptyAnomalyDetector( + String monitorItemName, String pattern) { + super(monitorItemName, pattern); + // TODO Auto-generated constructor stub + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleSingleTypeAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + + boolean anomalous = !(strategyData.getData().isEmpty()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + this.getMonitorItemName() + " IS EMPTY"+ "\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringNotEqualToAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringNotEqualToAnomalyDetector.java new file mode 100644 index 0000000..9e51a05 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringNotEqualToAnomalyDetector.java @@ -0,0 +1,62 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleStringNotEqualToAnomalyDetector extends + SingleCycleSingleStringAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param pattern + */ + public SingleCycleSingleStringNotEqualToAnomalyDetector(String monitorItemName, + String pattern) { + super(monitorItemName, pattern); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + + boolean anomalous = + !(strategyData.getData().equals(this.getPattern())); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " NOT EQUAL TO '" + getPattern() + "'\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringNotMatchRegExpAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringNotMatchRegExpAnomalyDetector.java new file mode 100644 index 0000000..e74075c --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleStringNotMatchRegExpAnomalyDetector.java @@ -0,0 +1,65 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.sql.Date; +import java.text.SimpleDateFormat; +import java.util.regex.Pattern; + +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +/** + * @author guoyezhi + * + */ +public class SingleCycleSingleStringNotMatchRegExpAnomalyDetector extends + SingleCycleSingleStringAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param monitorItemName + * @param pattern + */ + public SingleCycleSingleStringNotMatchRegExpAnomalyDetector( + String monitorItemName, + String pattern) { + super(monitorItemName, pattern); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + long timestamp = strategyData.getTimestamp(); + + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + + Pattern p = Pattern.compile(this.getPattern()); + boolean anomalous = + !(p.matcher(strategyData.getData()).find()); + if (anomalous) { + alarmLevel = MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + alarmInfo = + getMonitorItemName() + " NOT MATCH '" + getPattern() + "'\n" + + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + strategyData.getData(); + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + } + +} diff --git a/src/java/hma/monitor/strategy/detect/SingleCycleSingleTypeAnomalyDetector.java b/src/java/hma/monitor/strategy/detect/SingleCycleSingleTypeAnomalyDetector.java new file mode 100644 index 0000000..6b3622f --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/SingleCycleSingleTypeAnomalyDetector.java @@ -0,0 +1,50 @@ +/** + * + */ +package hma.monitor.strategy.detect; + +import java.util.Date; +import java.util.NavigableMap; + +import hma.monitor.MonitorStrategy; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyWorker; + +/** + * @author guoyezhi + * + */ +public abstract class SingleCycleSingleTypeAnomalyDetector extends + SingleCycleAnomalyDetector { + + /** + * @param monitorItemName + */ + public SingleCycleSingleTypeAnomalyDetector(String monitorItemName) { + super(monitorItemName); + } + + abstract protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData); + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.AnomalyDetector#detect(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public AnomalyDetectionResult detect(MonitorStrategyWorker worker) { + + MonitorStrategy strategy = worker.getStrategy(); + NavigableMap dataProcessingCentre = + strategy.getDataProcessingCentre(); + + synchronized (dataProcessingCentre) { + Date acquisitionTime = worker.getDataAcquisitionTimepoint(); + return this.visitMonitorStrategyData( + strategy.retrieveSingleCycleStrategyData( + getMonitorItemName(), acquisitionTime)); + } + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_LiveDatanodeHeartBeatIntervalDetector.java b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_LiveDatanodeHeartBeatIntervalDetector.java new file mode 100644 index 0000000..1703841 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_LiveDatanodeHeartBeatIntervalDetector.java @@ -0,0 +1,194 @@ +/** + * + */ +package hma.monitor.strategy.detect.custom; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.detect.AnomalyDetectionResult; +import hma.monitor.strategy.detect.MultipleCyclesCompoundTypeAnomalyDetector; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; + +import hma.util.InspectionDataInjector; +import java.sql.SQLException; +import java.text.ParseException; + +/** + * @author guoyezhi + * + */ +public class HMA_HDFS_LiveDatanodeHeartBeatIntervalDetector extends + MultipleCyclesCompoundTypeAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + + /** + * @param monitorItemName + * @param cycles + */ + public HMA_HDFS_LiveDatanodeHeartBeatIntervalDetector( + String monitorItemName, int cycles) { + super(monitorItemName, cycles); + } + + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.MultipleCyclesCompoundTypeAnomalyDetector#visitMultipleCyclesMonitorStrategyDataCollection(java.util.NavigableMap) + */ + @SuppressWarnings("unchecked") + @Override + protected AnomalyDetectionResult visitMultipleCyclesMonitorStrategyDataCollection( + NavigableMap multipleCyclesDataCollection) { + + final String itemName = "HDFS_RPC_LiveDatanodes_LastContact"; + if (!getMonitorItemName().equals(itemName)) { + throw new RuntimeException("Configuration Error"); + } + + if (multipleCyclesDataCollection.size() < this.getCycles()) { + return null; + } + + StringBuilder detectionInfoBuilder = new StringBuilder(); + MonitorStrategyData> lastStrategyData = + (MonitorStrategyData>) + multipleCyclesDataCollection.lastEntry().getValue().get(itemName); + if (lastStrategyData == null || lastStrategyData.getData() == null) { + return null; + } + detectionInfoBuilder.append( + "[" + dateFormat.format(multipleCyclesDataCollection.lastKey()) + + "] Live Datanode HeartBeat Interval Status List :\n"); + Iterator> iter = + lastStrategyData.getData().entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String node = entry.getKey(); + long interval = entry.getValue(); + detectionInfoBuilder.append("\t" + node + " -> " + interval + "\n"); + } + String detectionInfo = detectionInfoBuilder.toString(); + + Configuration conf = MonitorManager.getGlobalConf(); + long hbIntervalThres = conf.getLong( + "datanode.heartbeat.interval.alarm.threshold", + 60); + double incidenceThres = conf.getFloat( + "heartbeat.interval.anomaly.incidence.alarm.threshold", + (float) 50.0); + int dnCntWarningThres = conf.getInt( + "heartbeat.interval.anomalous.datanode.count.warning.level.alarm.threshold", + 10); + int dnCntCriticalThres = conf.getInt( + "heartbeat.interval.anomalous.datanode.count.critical.level.alarm.threshold", + 40); + + int i = this.getCycles(); + Date d = multipleCyclesDataCollection.lastKey(); + Map counter = new HashMap(); + while ((i > 0) && (d != null)) { + + MonitorStrategyData strategyData = + multipleCyclesDataCollection.get(d).get(itemName); + if (strategyData == null || strategyData.getData() == null) { + continue; + } + Map toDetect = + (Map) strategyData.getData(); + + for (String node : toDetect.keySet()) { + Long interval = toDetect.get(node); + if (interval == null) continue; + if (interval >= hbIntervalThres) { + Integer occurences = counter.get(node); + if (occurences == null) { + counter.put(node, 1); + } else { + counter.put(node, occurences + 1); + } + } + } + + i--; + d = multipleCyclesDataCollection.lowerKey(d); + + } + + boolean anomalous = false; + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + String alarmInfo = null; + StringBuilder alarmInfoBuilder = new StringBuilder(); + Map detectionResult = new HashMap(); + + if ((i == 0) && (counter.size() > 0)) { + + for (String node : counter.keySet()) { + int occurences = counter.get(node); + double incidence = + 100.0 * occurences / this.getCycles(); + if (incidence > incidenceThres) { + detectionResult.put(node, occurences); + } + } + + if (detectionResult.size() >= dnCntCriticalThres) { + anomalous = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + } else if (detectionResult.size() >= dnCntWarningThres) { + anomalous = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + } else if (detectionResult.size() > 0) { + anomalous = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_INFO; + } + + if (anomalous) { + alarmInfoBuilder.append( + "心跳间隔过高 DN 总数：" + detectionResult.size() + "\n"); + for (String anomaly : detectionResult.keySet()) { + long interval = lastStrategyData.getData().get(anomaly); + alarmInfoBuilder.append("\t" + anomaly + " -> " + interval + "\n"); + } + alarmInfo = alarmInfoBuilder.toString(); + } + } + + String clusterName = MonitorManager.getMonitoredClusterName(); + String strategyName = multipleCyclesDataCollection.lastEntry().getValue().getStrategyName(); + String detectionTimestamp = dateFormat.format(new Date()); + String infoLevel = MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue( + alarmLevel); + String infoContent = detectionResult.size() + ""; + + + try { + new InspectionDataInjector(clusterName,strategyName,detectionTimestamp,infoLevel,infoContent); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ParseException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_LiveDatanodeXceiverUsageDetector.java b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_LiveDatanodeXceiverUsageDetector.java new file mode 100644 index 0000000..e1ffc2d --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_LiveDatanodeXceiverUsageDetector.java @@ -0,0 +1,197 @@ +/** + * + */ +package hma.monitor.strategy.detect.custom; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.detect.AnomalyDetectionResult; +import hma.monitor.strategy.detect.MultipleCyclesCompoundTypeAnomalyDetector; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; + +import java.sql.SQLException; +import java.text.ParseException; +import hma.util.InspectionDataInjector; +import java.text.DecimalFormat; + +/** + * @author guoyezhi + * + */ +public class HMA_HDFS_LiveDatanodeXceiverUsageDetector extends + MultipleCyclesCompoundTypeAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + + /** + * @param monitorItemName + * @param cycles + */ + public HMA_HDFS_LiveDatanodeXceiverUsageDetector(String monitorItemName, + int cycles) { + super(monitorItemName, cycles); + } + + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.MultipleCyclesCompoundTypeAnomalyDetector#visitMultipleCyclesMonitorStrategyDataCollection(java.util.NavigableMap) + */ + @SuppressWarnings("unchecked") + @Override + protected AnomalyDetectionResult visitMultipleCyclesMonitorStrategyDataCollection( + NavigableMap multipleCyclesDataCollection) { + + final String itemName = "HDFS_RPC_LiveDatanodes_XceiverUsage"; + if (!getMonitorItemName().equals(itemName)) { + throw new RuntimeException("Configuration Error"); + } + + if (multipleCyclesDataCollection.size() < this.getCycles()) { + return null; + } + + StringBuilder detectionInfoBuilder = new StringBuilder(); + MonitorStrategyData> lastStrategyData = + (MonitorStrategyData>) + multipleCyclesDataCollection.lastEntry().getValue().get(itemName); + if (lastStrategyData == null || lastStrategyData.getData() == null) { + return null; + } + detectionInfoBuilder.append( + "[" + dateFormat.format(multipleCyclesDataCollection.lastKey()) + + "] Live Datanode Xceiver Usage Status List :\n"); + Iterator> iter = + lastStrategyData.getData().entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String node = entry.getKey(); + long usage = entry.getValue(); + detectionInfoBuilder.append("\t" + node + " -> " + usage + "\n"); + } + String detectionInfo = detectionInfoBuilder.toString(); + + Configuration conf = MonitorManager.getGlobalConf(); + long xceiverUsageThres = conf.getLong( + "datanode.xceiver.urate.alarm.threshold", + 600); + int dnCntWarningThres = conf.getInt( + "xceiver.urate.anomalous.datanode.count.warning.level.alarm.threshold", + 10); + int dnCntCriticalThres = conf.getInt( + "xceiver.urate.anomalous.datanode.count.critical.level.alarm.threshold", + 40); + + int i = this.getCycles(); + Date d = multipleCyclesDataCollection.lastKey(); + Set anomalies = lastStrategyData.getData().keySet(); + + double total = anomalies.size() * 1.0 ; + boolean slave_rate = conf.getBoolean("slave.rate",true) ; + float slaveWarnThres = conf.getFloat("dn.xceiver.warn.percent.thres",(float)0.2); + float slaveCriticalThres = conf.getFloat("dn.xceiver.critical.percent.thres",(float)0.3); + while ((i > 0) && (d != null) && (anomalies.size() > 0)) { + + MonitorStrategyData strategyData = + multipleCyclesDataCollection.get(d).get(itemName); + if (strategyData == null || strategyData.getData() == null) { + return null; + } + Map toDetect = + (Map) strategyData.getData(); + Map detected = new HashMap(); + + Iterator it = anomalies.iterator(); + while (it.hasNext()) { + String node = it.next(); + Long usage = toDetect.get(node); + if (usage == null) continue; + if (usage >= xceiverUsageThres) { + detected.put(node, usage); + } + } + + i--; + d = multipleCyclesDataCollection.lowerKey(d); + anomalies = detected.keySet(); + + } + + boolean anomalous = false; + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + String alarmInfo = null; + StringBuilder alarmInfoBuilder = new StringBuilder(); + + double anomalieNum = anomalies.size() * 1.0; + DecimalFormat df1 = new DecimalFormat("0.00"); + float nownorate = Float.valueOf(df1.format( anomalieNum / total)).floatValue(); + // System.out.println("masuhua dn xceiver total:"+total+" slave_rate:"+slave_rate+" no num:"+anomalieNum+" rate:"+nownorate+" slaveThres:"+slaveThres); + if ((i == 0) && (anomalies.size() > 0)) { + if ( slave_rate ){ + if ( nownorate >= slaveCriticalThres){ + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + }else{ + if(nownorate >= slaveWarnThres) { + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + }else{ + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_INFO; + } + } + }else{ + if ( anomalies.size() >= dnCntCriticalThres ){ + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + } else { + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_INFO; + } + } + anomalous = true; + alarmInfoBuilder.append( + "Xceiver 句柄过高 DN 总数：" + anomalies.size() + " 占比:" + nownorate+"\n"); + for (String anomaly : anomalies) { + long usage = lastStrategyData.getData().get(anomaly); + alarmInfoBuilder.append("\t" + anomaly + " -> " + usage + "\n"); + } + alarmInfo = alarmInfoBuilder.toString(); + } + + String clusterName = MonitorManager.getMonitoredClusterName(); + String strategyName = multipleCyclesDataCollection.lastEntry().getValue().getStrategyName(); + String detectionTimestamp = dateFormat.format(new Date()); + String infoLevel = MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue( + alarmLevel); + String infoContent = anomalies.size() + ""; + //System.out.println ("kain's inspection log: " + clusterName + " " + strategyName + " " + detectionTimestamp + " " + infoLevel + " " + infoContent ); + + try { + new InspectionDataInjector(clusterName,strategyName,detectionTimestamp,infoLevel,infoContent); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ParseException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_NSnodeRPCAvailabilityDetector.java b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_NSnodeRPCAvailabilityDetector.java new file mode 100644 index 0000000..35846cc --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_NSnodeRPCAvailabilityDetector.java @@ -0,0 +1,96 @@ +/** + * + */ +package hma.monitor.strategy.detect.custom; + +import hma.monitor.MonitorManager; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.detect.AnomalyDetectionResult; +import hma.monitor.strategy.detect.SingleCycleSingleStringAnomalyDetector; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * @author guoyezhi + * + */ +public class HMA_HDFS_NSnodeRPCAvailabilityDetector extends + SingleCycleSingleStringAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + + /** + * @param monitorItemName + * @param pattern + */ + public HMA_HDFS_NSnodeRPCAvailabilityDetector(String monitorItemName, + String pattern) { + super(monitorItemName, pattern); + } + + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleSingleTypeAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + final String itemName = "HDFS_RPC_FileSystem_ListRootPathEntries"; + if (!getMonitorItemName().equals(itemName)) { + throw new RuntimeException("Configuration Error"); + } + + if (strategyData == null || + (strategyData.getData() == null)) { + return null; + } + String statuses = (String) strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + boolean anomalous = false; + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" + statuses; + + long threshold = + MonitorManager.getGlobalConf().getLong( + "namenode.rpc.timeout.alarm.threshold", 300000); + long currentTime = System.currentTimeMillis(); + anomalous = ((currentTime - timestamp) > threshold); + if (anomalous) { + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + alarmInfo = + "HDFS服务不可用" + ((currentTime - timestamp) / 1000) + "秒！"; + } else { + this.setPattern( + MonitorManager.getMonitoredClusterName().toLowerCase()); + anomalous = + !(statuses.toLowerCase().contains(this.getPattern())); + if (anomalous) { + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + alarmInfo = + "未探测到集群标志文件，HDFS根目录可能发生了误删除！\n" + + getMonitorItemName() + + " (Collected at: " + + dateFormat.format(new Date(timestamp)) + + ") :\n" + + statuses; + } + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_NameNodeWriteAndReadAvailabilityDetector.java b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_NameNodeWriteAndReadAvailabilityDetector.java new file mode 100644 index 0000000..56cd694 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_NameNodeWriteAndReadAvailabilityDetector.java @@ -0,0 +1,122 @@ + +package hma.monitor.strategy.detect.custom; + +import hma.monitor.MonitorManager; +import hma.monitor.collection.custom.HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm_MonitorItemWorker.BenchmarkOP; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.detect.AnomalyDetectionResult; +import hma.monitor.strategy.detect.SingleCycleSingleStringAnomalyDetector; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; + +/** + * @author kain + * + */ +public class HMA_HDFS_NameNodeWriteAndReadAvailabilityDetector extends + SingleCycleSingleStringAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + + /** + * @param monitorItemName + * @param pattern + */ + public HMA_HDFS_NameNodeWriteAndReadAvailabilityDetector(String monitorItemName, + String pattern) { + super(monitorItemName, pattern); + } + + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleSingleTypeAnomalyDetector#visitMonitorStrategyData(hma.monitor.strategy.MonitorStrategyData) + */ + @Override + protected AnomalyDetectionResult visitMonitorStrategyData( + MonitorStrategyData strategyData) { + + final String itemName = "HDFS_RPC_FileSystem_MkdirAndPutAndCatAndRm"; + if (!getMonitorItemName().equals(itemName)) { + throw new RuntimeException("Configuration Error"); + } + + if (strategyData == null || + (strategyData.getData() == null)) { + return null; + } + System.out.println("kain's log write readwrite strategy enter"); + Object tmpObject = (Object)strategyData.getData(); + Map resMap = (Map) tmpObject; + long timestamp = strategyData.getTimestamp(); + + boolean anomalous = false; + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + String alarmInfo = null; + String dectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + getMonitorItemName() + " :\n" ; + + long threshold = + MonitorManager.getGlobalConf().getLong( + "namenode.rpc.timeout.alarm.threshold", 300000); + long currentTime = System.currentTimeMillis(); + anomalous = ((currentTime - timestamp) > threshold); + System.out.println("kain's log write readwrite strategy anomlous val " + anomalous ); + if (anomalous) { + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + alarmInfo = + "HDFS服务不可用" + ((currentTime - timestamp) / 1000) + "秒！"; + } else { + String problemOP = ""; + for(Map.Entry + opEntry:resMap.entrySet() ){ + System.out.println("kain's log write readwrite strategy " + opEntry.getKey().toString() + " " + opEntry.getValue()); + if(!opEntry.getValue()){ + anomalous = true; + + switch(opEntry.getKey()){ + case mkdir: + problemOP = "建立目录"; + break; + case putFile: + problemOP = "上传文件"; + break; + case catFile: + problemOP = "cat文件"; + break; + case rmAllB: + problemOP = "删除文件"; + break; + + } + + + } + } + + if (anomalous) { + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + alarmInfo = + problemOP + "功能不可用！\n" + + getMonitorItemName() + + " (Collected at: " + + dateFormat.format(new Date(timestamp)) + + ") :\n" ; + } + } + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, dectionInfo); + + } + + +} diff --git a/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_RecentlyRsyncedMetadataFileStatusDetector.java b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_RecentlyRsyncedMetadataFileStatusDetector.java new file mode 100644 index 0000000..2ead641 --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/custom/HMA_HDFS_RecentlyRsyncedMetadataFileStatusDetector.java @@ -0,0 +1,198 @@ +/** + * + */ +package hma.monitor.strategy.detect.custom; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.detect.AnomalyDetectionResult; +import hma.monitor.strategy.detect.SingleCycleCompoundTypeAnomalyDetector; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.fs.FileStatus; + +import java.sql.SQLException; +import java.text.ParseException; +import hma.util.InspectionDataInjector; + +/** + * @author guoyezhi + * + */ +public class HMA_HDFS_RecentlyRsyncedMetadataFileStatusDetector extends + SingleCycleCompoundTypeAnomalyDetector { + + private static final SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + + /** + * @param monitorItemName + */ + public HMA_HDFS_RecentlyRsyncedMetadataFileStatusDetector(String monitorItemName) { + super(monitorItemName); + } + + + /* (non-Javadoc) + * @see hma.monitor.strategy.detect.SingleCycleCompoundTypeAnomalyDetector#visitMonitorStrategyDataCollection(hma.monitor.strategy.MonitorStrategyDataCollection) + */ + @SuppressWarnings("unchecked") + @Override + protected AnomalyDetectionResult visitMonitorStrategyDataCollection( + MonitorStrategyDataCollection strategyDataCollection) { + + final String itemName = + "HDFS_RtExec_SecondaryNamenode_RecentlyRsyncedMetadataFileStatus"; + if (!getMonitorItemName().equals(itemName)) { + throw new RuntimeException("Configuration Error"); + } + + MonitorStrategyData strategyData = + strategyDataCollection.get(itemName); + if (strategyData == null || + (strategyData.getData() == null)) { + return null; + } + List statusList = + (List) strategyData.getData(); + long timestamp = strategyData.getTimestamp(); + + Configuration conf = MonitorManager.getGlobalConf(); + long newEditsFSizeThreshold = + conf.getLong("recently.rsynced.new.edits.file.size.alarm.threshold", + 400); + long editsMTimeThreshold = + conf.getLong("recently.rsynced.edits.file.mtime.alarm.threshold", + 45); + long fsimageMTimeThreshold = + conf.getLong("recently.rsynced.fsimage.file.mtime.alarm.threshold", + 120); + long currentTime = System.currentTimeMillis(); + + boolean anomalous = false; + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + StringBuilder alarmInfoBuilder = new StringBuilder(); + StringBuilder detectionInfoBuilder = new StringBuilder(); + StringBuilder additionalInfoBuilder = new StringBuilder(); + + int i = 0; + Map> checkPointMap = new LinkedHashMap>(); + for (FileStatus status : statusList) { + + String name = status.getPath().getName(); + long mtime = status.getModificationTime(); + String owner = status.getOwner(); + + if (name.endsWith("edits")) { + + detectionInfoBuilder.append( + "\tedits file is rsynced at " + + dateFormat.format(new Date(mtime)) + ".\n"); + + if (currentTime - mtime >= editsMTimeThreshold * 60 * 1000) { + anomalous = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + alarmInfoBuilder.append( + (++i) + ") " + owner + " : edits file NOT UPDATED more than " + + editsMTimeThreshold + " mins. "); + if (additionalInfoBuilder.length() > 0) { + additionalInfoBuilder.append(","); + } + additionalInfoBuilder.append(owner); + } + + } else if (name.endsWith("edits.new")) { + + detectionInfoBuilder.append( + "\tedits.new file size = " + status.getLen() + "\n"); + + if (status.getLen() >= (newEditsFSizeThreshold * 1024 * 1024)) { + anomalous = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + alarmInfoBuilder.append( + (++i) + ") " + owner + " : edits.new file size >= " + + newEditsFSizeThreshold + "MB. "); + if (additionalInfoBuilder.length() > 0) { + additionalInfoBuilder.append(","); + } + additionalInfoBuilder.append(owner); + } + + } else if (name.endsWith("fsimage")) { + + detectionInfoBuilder.append( + "\tfsimage file is rsynced at " + + dateFormat.format(new Date(mtime)) + ".\n"); + + if (currentTime - mtime >= fsimageMTimeThreshold * 60 * 1000) { + anomalous = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + alarmInfoBuilder.append( + (++i) + ") " + owner + " : fsimage file NOT UPDATED more than " + + fsimageMTimeThreshold + " mins. "); + if (additionalInfoBuilder.length() > 0) { + additionalInfoBuilder.append(","); + } + additionalInfoBuilder.append(owner); + } + + } + if (!checkPointMap.containsKey(owner)){ + Map metaInfo = new HashMap(); + checkPointMap.put(owner,metaInfo); + } + checkPointMap.get(owner).put(name,Long.toString(status.getLen()) + ";" + + Long.toString(mtime)); + //System.out.println(dateFormat.format(new Date()) + " kain's log about chkpoit status gen : " + "machine is : " + // + owner + " meta : " + name + " size&mtime: " + Long.toString(status.getLen()) + // + " " + Long.toString(mtime)); + } + + //new HDFS_Namenode_CheckpointStatusDisplay().CheckpointStatusDisplayer(checkPointMap); + + String alarmInfo = + alarmInfoBuilder.toString().trim() + "\n" + detectionInfoBuilder.toString(); + String detectionInfo = + "[" + dateFormat.format(new Date(timestamp)) + "] " + + "Recently-rsynced meta file status :\n" + + detectionInfoBuilder.toString(); + + String clusterName = MonitorManager.getMonitoredClusterName(); + String strategyName = strategyDataCollection.getStrategyName(); + String detectionTimestamp = dateFormat.format(new Date(timestamp)); + String infoLevel = MonitorStrategyAttachedTaskAdapter.getAlarmLevelStringFromValue(alarmLevel); + String infoContent = (alarmLevel == + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED ? "Normal":"AbNormal"); + + try { + new InspectionDataInjector(clusterName,strategyName,detectionTimestamp,infoLevel,infoContent); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ParseException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + //System.out.println("kain's inspection's log : " + clusterName + " " + strategyName + " " + detectionTimestamp + " " + infoLevel + " " + infoContent); + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + + } + +} diff --git a/src/java/hma/monitor/strategy/detect/custom/HMA_MapRed_ComputingFunctionAvailabilityDetector.java b/src/java/hma/monitor/strategy/detect/custom/HMA_MapRed_ComputingFunctionAvailabilityDetector.java new file mode 100644 index 0000000..603fe0f --- /dev/null +++ b/src/java/hma/monitor/strategy/detect/custom/HMA_MapRed_ComputingFunctionAvailabilityDetector.java @@ -0,0 +1,164 @@ +/** + * + */ +package hma.monitor.strategy.detect.custom; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.monitor.collection.custom.MapRed_RPC_ComputingStatus_AvailabilityProbingJob_MonitorItemWorker.HMAProbingJobResult; +import hma.monitor.strategy.MonitorStrategyData; +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.detect.AnomalyDetectionResult; +import hma.monitor.strategy.detect.MultipleCyclesCompoundTypeAnomalyDetector; +import hma.monitor.strategy.trigger.task.MonitorStrategyAttachedTaskAdapter; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.NavigableMap; + +/** + * @author guoyezhi + * + */ +public class HMA_MapRed_ComputingFunctionAvailabilityDetector extends + MultipleCyclesCompoundTypeAnomalyDetector { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + + /** + * @param monitorItemName + * @param cycles + */ + public HMA_MapRed_ComputingFunctionAvailabilityDetector( + String monitorItemName, int cycles) { + super(monitorItemName, cycles); + } + + + @SuppressWarnings("unchecked") + @Override + protected AnomalyDetectionResult visitMultipleCyclesMonitorStrategyDataCollection( + NavigableMap multipleCyclesDataCollection) { + + final String jobItemName = + "MapRed_RPC_ComputingStatus_AvailabilityProbingJob"; + + if (multipleCyclesDataCollection.size() < this.getCycles()) { + return null; + } + + Configuration conf = MonitorManager.getGlobalConf(); + int involvedCycles = conf.getInt("probing.job.involved.cycle.num", 10); + if (this.getCycles() < involvedCycles) { + throw new RuntimeException("Configuration Error"); + } + + HMAProbingJobResult jobResult = null; + StringBuilder alarmInfoBuilder = new StringBuilder(); + StringBuilder detectionInfoBuilder = new StringBuilder(); + String PhyQueueName = MonitorManager.getGlobalConf().get("mapred.abaci.phyqueuename", "null"); + List queueList = new ArrayList(); + if (!PhyQueueName.equals("null")){ + String QueueNames [] = PhyQueueName.split(";"); + for(String queueName:QueueNames){ + queueList.add(jobItemName + ":" + queueName); + } + } + queueList.add(jobItemName); + + boolean anomalous = false; + int alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_DETECTION_PASSED; + for (int i = 0; i < queueList.size(); i++ ){ + String nowJobItemName = queueList.get(i); + //System.out.println("kain's log about mutiqueue monitor, nowQueueName is : " + nowJobItemName); + + for (Date d : multipleCyclesDataCollection.keySet()) { + MonitorStrategyData strategyData = + (MonitorStrategyData) + multipleCyclesDataCollection.get(d).get(nowJobItemName); + + if (strategyData == null || strategyData.getData() == null) { + continue; + } + HMAProbingJobResult tmp = strategyData.getData(); + if (jobResult == null || + jobResult.getJobSubmitTime().before(tmp.getJobSubmitTime())) { + /* + * pick up the latest submitted job + */ + jobResult = tmp; + } + } + + + + detectionInfoBuilder.append( + "[" + dateFormat.format(multipleCyclesDataCollection.lastKey()) + "] " + + "Job: " + jobResult.getJobID() + "\n\t" + + "Name: " + jobResult.getJobName() + "\n\t" + + "Queue: " + jobResult.getQueueName() + "\n\t" + + "Successful: " + jobResult.isSuccessful() + "\n\t" + + "Submit Time: " + jobResult.getJobSubmitTime() + "\n\t" + + "Complete Time: " + jobResult.getJobCompleteTime() + "\n" ); + + long currTimestamp = System.currentTimeMillis(); + long jobSubmitTime = jobResult.getJobSubmitTime().getTime(); + long timeoutWarningThres = 1000 * conf.getLong( + "probing.job.timeout.warning.level.alarm.threshold", + 600); + long timeoutCriticalThres = 1000 * conf.getLong( + "probing.job.timeout.critical.level.alarm.threshold", + 900); + if (currTimestamp - jobSubmitTime >= timeoutWarningThres) { + + if (currTimestamp - jobSubmitTime >= timeoutCriticalThres) { + anomalous = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + } else { + anomalous = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_WARN; + } + + alarmInfoBuilder.append("Queue:" + jobResult.getQueueName() +" Probing Job Timeout!\n"); + + alarmInfoBuilder.append( + "\t" + + "Job: " + jobResult.getJobID() + "\n\t" + + "Name: " + jobResult.getJobName() + "\n\t" + + "Queue: " + jobResult.getQueueName() + "\n\t" + + "Successful: " + jobResult.isSuccessful() + "\n\t" + + "Submit Time: " + jobResult.getJobSubmitTime() + "\n\t" + + "Complete Time: " + jobResult.getJobCompleteTime() + "\n"); + + } else if (!jobResult.isSuccessful()) { + anomalous = true; + alarmLevel = + MonitorStrategyAttachedTaskAdapter.ALARM_LEVEL_CRITICAL; + alarmInfoBuilder.append( + "Queue:" + jobResult.getQueueName() + " " + + "Probing Job Failed!\n\t" + + "Job: " + jobResult.getJobID() + "\n\t" + + "Name: " + jobResult.getJobName() + "\n\t" + + "Queue: " + jobResult.getQueueName() + "\n\t" + + "Successful: " + jobResult.isSuccessful() + "\n\t" + + "Submit Time: " + jobResult.getJobSubmitTime() + "\n\t" + + "Complete Time: " + jobResult.getJobCompleteTime() + "\n"); + } + } + + String alarmInfo = alarmInfoBuilder.toString().trim() + "\n" ; + String detectionInfo = detectionInfoBuilder.toString(); + + return new AnomalyDetectionResult( + anomalous, alarmLevel, alarmInfo, detectionInfo); + + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/ContinuousTypeTaskTrigger.java b/src/java/hma/monitor/strategy/trigger/ContinuousTypeTaskTrigger.java new file mode 100644 index 0000000..073adab --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/ContinuousTypeTaskTrigger.java @@ -0,0 +1,105 @@ +/** + * + */ +package hma.monitor.strategy.trigger; + +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyWorker; + +import java.util.Date; +import java.util.NavigableMap; + +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; + +/** + * @author guoyezhi + * + */ +public class ContinuousTypeTaskTrigger implements TaskTrigger { + + private int threshold = 0; + + public ContinuousTypeTaskTrigger(Element conf) { + + NodeList subFieldNodes = conf.getChildNodes(); + + String str = null; + for (int i = 0; i < subFieldNodes.getLength(); i++) { + Node subFieldNode = subFieldNodes.item(i); + if (!(subFieldNode instanceof Element)) + continue; + Element subField = (Element)subFieldNode; + if ("continuity_threshold".equals(subField.getTagName()) + && subField.hasChildNodes()) { + str = ((Text)subField.getFirstChild()).getData().trim(); + } + } + if (str != null) + this.threshold = Integer.parseInt(str); + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.trigger.TaskTrigger#visitMonitorStrategyWorker(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public boolean visitMonitorStrategyWorker(MonitorStrategyWorker worker) { + + if (threshold < 1) { + throw new RuntimeException(); + } + + NavigableMap dataProcessingCentre = + worker.getStrategy().getDataProcessingCentre(); + Date timestamp = worker.getDataAcquisitionTimepoint(); + + NavigableMap view = + dataProcessingCentre.headMap(timestamp, true); + System.err.println("\tview.size() = " + view.size()); + if (view.size() < threshold) + return false; + + MonitorStrategyDataCollection dc = view.get(timestamp); + for (int i = threshold; dc != null && i > 0; i--) { + /* + System.err.println("\t" + new Date( + dc.get("NameNode_ListHDFSRootPath").getTimestamp())); + */ + if (dc.isAnomalous() == false) { + System.err.println("\ti = " + i + ", dc.isAnomalous() == false"); + return false; + } + timestamp = view.lowerKey(timestamp); + if (timestamp != null) { + dc = view.get(timestamp); + } + } + + /* + System.err.println("\tview.size() = " + view.size()); + Iterator iter = view.values().iterator(); + System.err.println("\tview.firstKey() = " + view.firstKey()); + for (int i = threshold; iter.hasNext() && i > 0; i--) { + MonitorStrategyDataCollection dc = iter.next(); + System.err.println("\t" + new Date( + dc.get("NameNode_ListHDFSRootPath").getTimestamp())); + System.err.println("\t" + dc.get("NameNode_ListHDFSRootPath").getData()); + if (dc.isAnomalous() == false) { + System.err.println("\ti = " + i + ", dc.isAnomalous() == false"); + return false; + } + } + */ + + return true; + } + + /** + * @param args + */ + public static void main(String[] args) { + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/EverytimeTypeTaskTrigger.java b/src/java/hma/monitor/strategy/trigger/EverytimeTypeTaskTrigger.java new file mode 100644 index 0000000..7e20c5a --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/EverytimeTypeTaskTrigger.java @@ -0,0 +1,46 @@ +/** + * + */ +package hma.monitor.strategy.trigger; + +import java.util.Date; +import java.util.SortedMap; + +import org.w3c.dom.Element; + +import hma.monitor.strategy.MonitorStrategyDataCollection; +import hma.monitor.strategy.MonitorStrategyWorker; + +/** + * @author guoyezhi + * + */ +public class EverytimeTypeTaskTrigger implements TaskTrigger { + + public EverytimeTypeTaskTrigger(Element element) { + // nothing needed to do + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.trigger.TaskTrigger#visitMonitorStrategyWorker(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public boolean visitMonitorStrategyWorker(MonitorStrategyWorker worker) { + + SortedMap dataProcessingCentre = + worker.getStrategy().getDataProcessingCentre(); + Date timestamp = worker.getDataAcquisitionTimepoint(); + MonitorStrategyDataCollection dataCollection = + dataProcessingCentre.get(timestamp); + return dataCollection.isAnomalous(); + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/NeverTypeTaskTrigger.java b/src/java/hma/monitor/strategy/trigger/NeverTypeTaskTrigger.java new file mode 100644 index 0000000..9e1f4a9 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/NeverTypeTaskTrigger.java @@ -0,0 +1,39 @@ +/** + * + */ +package hma.monitor.strategy.trigger; + +import org.w3c.dom.Element; + +import hma.monitor.strategy.MonitorStrategyWorker; + +/** + * @author guoyezhi + * + */ +public class NeverTypeTaskTrigger implements TaskTrigger { + + /** + * + */ + public NeverTypeTaskTrigger(Element conf) { + // TODO Auto-generated constructor stub + } + + /* (non-Javadoc) + * @see hma.monitor.strategy.trigger.TaskTrigger#visitMonitorStrategyWorker(hma.monitor.strategy.MonitorStrategyWorker) + */ + @Override + public boolean visitMonitorStrategyWorker(MonitorStrategyWorker worker) { + return false; // always return false + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/TaskTrigger.java b/src/java/hma/monitor/strategy/trigger/TaskTrigger.java new file mode 100644 index 0000000..1a35247 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/TaskTrigger.java @@ -0,0 +1,16 @@ +/** + * + */ +package hma.monitor.strategy.trigger; + +import hma.monitor.strategy.MonitorStrategyWorker; + +/** + * @author guoyezhi + * + */ +public interface TaskTrigger { + + boolean visitMonitorStrategyWorker(MonitorStrategyWorker worker); + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedCallbackReporter.java b/src/java/hma/monitor/strategy/trigger/task/AttachedCallbackReporter.java new file mode 100644 index 0000000..a03a601 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedCallbackReporter.java @@ -0,0 +1,159 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.web.HMAJspHelper; + +import java.util.Date; +import java.util.HashMap; +import java.io.IOException; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import java.net.URL; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONArray; + + +/** + * @author chiwen01 + * + */ +public class AttachedCallbackReporter extends MonitorStrategyAttachedTask { + + private String callbackUrls[] = null; + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedCallbackReporter(String taskWorker, String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + this.callbackUrls = + MonitorManager.getGlobalConf().getStrings("alarm.callback.url"); + } + + private void doSendCallbackMsg(String message) { + try { + for (int i = 0; i < callbackUrls.length; i++) { + URL url = new URL(callbackUrls[i]); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setDoInput(true); + conn.setRequestMethod("POST"); + conn.setRequestProperty("User-Agent", "HMA"); + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + conn.connect(); + OutputStreamWriter alarmMessage = new OutputStreamWriter(conn.getOutputStream(), "UTF-8"); + alarmMessage.write(message); + alarmMessage.flush(); + alarmMessage.close(); + BufferedReader in = new BufferedReader( + new InputStreamReader(conn.getInputStream())); + String response; + while ((response = in.readLine()) != null) + System.out.println("callback server "+ callbackUrls[i]+ " return:" + response); + } + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + String monitorStrategyName = + taskArgs.getMonitorStrategyName(); + String alarmLevel = taskArgs.getAlarmLevel(); + String targetSystemName = taskArgs.getTargetSystemName(); + String additionalInfo = taskArgs.getAdditionalInfo(); + String key = null; + + Configuration conf = MonitorManager.getGlobalConf(); + String alarmThres = conf.get( + "callback.alarm.level.threshold", + "NOTICE"); + + if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + alarmLevel, alarmThres) <= 0) { + return; + } + + HashMap rule_data = new HashMap(); + JSONObject ns_data = new JSONObject(); + JSONObject data = new JSONObject(); + JSONObject postData = new JSONObject(); + JSONObject alarmInfo = new JSONObject(); + JSONArray ns_dataJsonArray = new JSONArray(); + JSONArray dataJsonArray = new JSONArray(); + StringBuilder rulenameBuilder = new StringBuilder(); + + try { + JSONObject additionalInfoJsonObject = new JSONObject(additionalInfo); + try { + additionalInfo = (String) additionalInfoJsonObject.get("additionalInfo"); + } catch(JSONException e) { + e.getCause(); + } + try { + key = (String) additionalInfoJsonObject.get("key"); + } catch(JSONException e) { + key = ""; + e.getCause(); + } + } catch (Exception e) { + key = ""; + e.printStackTrace(); + } + try { + int status = 1; + if (taskArgs.getFullInfo().startsWith("恢复正常")) { + status = 1; + } else { + status = 0; + } + rule_data.put("status", status); + rule_data.put("time", taskArgs.getTimestamp().getTime()/1000); + + rulenameBuilder.append(targetSystemName + ":"); + rulenameBuilder.append(alarmLevel + ":"); + rulenameBuilder.append(HMAJspHelper.getMonitorStrategyAlias(monitorStrategyName)); + ns_data.put("rule_name", rulenameBuilder.toString()); + ns_data.put("rule_data", rule_data); + alarmInfo.put("message", taskArgs.getKeyInfo()); + alarmInfo.put("key", key); + +// data.put("name_space", targetSystemName); + data.put("name_space", additionalInfo); + data.put("alarm_info", alarmInfo); + ns_dataJsonArray.put(0, ns_data); + data.put("ns_data", ns_dataJsonArray); + + postData.put("type", 2); + postData.put("token",""); + postData.put("name", "HMA"); + postData.put("time", new Date().getTime()/1000); + dataJsonArray.put(0,data); + postData.put("data", dataJsonArray); + } catch (JSONException jsonException) { + jsonException.getCause(); + } + + doSendCallbackMsg(postData.toString()); + + } +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedCustomizedWorker.java b/src/java/hma/monitor/strategy/trigger/task/AttachedCustomizedWorker.java new file mode 100644 index 0000000..6104816 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedCustomizedWorker.java @@ -0,0 +1,41 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +/** + * @author guoyezhi + * + */ +public class AttachedCustomizedWorker extends MonitorStrategyAttachedTask { + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedCustomizedWorker( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + // TODO Auto-generated method stub + + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedMailReporter.java b/src/java/hma/monitor/strategy/trigger/task/AttachedMailReporter.java new file mode 100644 index 0000000..0c04451 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedMailReporter.java @@ -0,0 +1,103 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.SimpleDateFormat; + +import org.apache.commons.mail.EmailException; +import org.apache.commons.mail.SimpleEmail; + +/** + * @author guoyezhi + * + */ +public class AttachedMailReporter extends MonitorStrategyAttachedTask { + + private String hostname = "127.0.0.1"; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param taskWorker + * @param workerArgs + * @param taskArgs + */ + public AttachedMailReporter( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments taskArgs) { + super(taskWorker, workerArgs, taskArgs); + BufferedReader read = null; + try { + Process proc = Runtime.getRuntime().exec("hostname"); + read = new BufferedReader(new InputStreamReader(proc.getInputStream())); + hostname = read.readLine(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } finally { + try { + read.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + private void doSendMail(String[] receivers, String subject, String body) { + + try { + SimpleEmail email = new SimpleEmail(); + for (int i = 0; i < receivers.length; i++){ + email.addTo(receivers[i]); + } + email.setCharset("gbk"); + //email.setFrom(System.getProperty("user.name") + "@"+ hostname); + String mfrom = "boyan5@staff.sina.com.cn"; + email.setFrom(mfrom); + email.setAuthentication("boyan5@staff.sina.com.cn", "ddzs%%017"); + email.setHostName("mail.staff.sina.com.cn"); + email.setSubject(subject); + email.setMsg(body); + email.send(); + } catch (EmailException ee) { + String tmp = ""; + for (int i = 0; i < receivers.length; i++){ + tmp = tmp + receivers[i]; + } + System.err.println("EmailException: mail is = " + tmp); + throw new RuntimeException(ee); + } + + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + String[] receivers = getWorkerArgs(); + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + + StringBuilder subjectBuilder = new StringBuilder(); + subjectBuilder.append("[" + taskArgs.getTargetSystemName() + "]"); + subjectBuilder.append("[" + taskArgs.getAlarmLevel() + "]"); + subjectBuilder.append("[" + taskArgs.getMonitorStrategyName() + "]"); + subjectBuilder.append("[" + taskArgs.getKeyInfo() + "]"); + subjectBuilder.append("[" + dateFormat.format(taskArgs.getTimestamp()) + "]"); + + StringBuilder bodyBuilder = new StringBuilder(); + bodyBuilder.append(subjectBuilder.toString() + "\n\n"); + bodyBuilder.append(taskArgs.getFullInfo()); + + doSendMail(receivers, subjectBuilder.toString(), bodyBuilder.toString()); + } + +} + diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedMobileReporter.bak b/src/java/hma/monitor/strategy/trigger/task/AttachedMobileReporter.bak new file mode 100644 index 0000000..6f8f28b --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedMobileReporter.bak @@ -0,0 +1,122 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; + +import java.io.DataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.Charset; +import java.text.SimpleDateFormat; + +/** + * @author guoyezhi + * + */ +public class AttachedMobileReporter extends MonitorStrategyAttachedTask { + + private String[] servers = null; + + private String hmaGSMsend = null; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param taskWorker + * @param workerArgs + * @param taskArgs + */ + public AttachedMobileReporter( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments taskArgs) { + super(taskWorker, workerArgs, taskArgs); + this.servers = + MonitorManager.getGlobalConf().getStrings("baidu.gsm.servers"); + this.hmaGSMsend = + System.getProperty("hma.home.dir") + "/conf/" + + MonitorManager.getMonitoredClusterName() + "/gsmsend.hma"; + } + + private void doSendSM(String[] receivers, String message) { + + Runtime rt = Runtime.getRuntime(); + + for (int i = 0; i < receivers.length; i++) { + for (int j = 0; j < servers.length; j++) { + try { + for (String str : new String[] { + "gsmsend", "-s", servers[j].trim(), + receivers[i].trim() + "@" + '"' + message + '"' }) { + System.out.print(str + " "); + } + System.out.println(); + + rt.exec(new String[] { + hmaGSMsend, + "-s", servers[j].trim(), + receivers[i].trim(), + message }); + + break; + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + String[] receivers = getWorkerArgs(); + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + String infoLevel = taskArgs.getAlarmLevel(); + Configuration conf = MonitorManager.getGlobalConf(); + String alarmThres = conf.get( + "mobile.reporter.alarm.level.threshold", + "NOTICE"); + + if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + infoLevel, alarmThres) <= 0) { + return; + } + + StringBuilder msgBuilder = new StringBuilder(); + msgBuilder.append("[HMA]"); + msgBuilder.append("[" + taskArgs.getTargetSystemName() + "]"); + msgBuilder.append("[" + taskArgs.getAlarmLevel() + "]"); + msgBuilder.append("[" + taskArgs.getMonitorStrategyName() + "]"); + msgBuilder.append("[" + taskArgs.getKeyInfo() + "]"); + msgBuilder.append("[" + dateFormat.format(taskArgs.getTimestamp()) + "]"); + + doSendSM(receivers, msgBuilder.toString()); + + } + + public static void main(String[] args) throws IOException { + + String message = "中文短信测试"; + byte[] buff = + Charset.forName("gbk").encode(message).array(); + message = ""; + DataOutputStream out = new DataOutputStream( + new FileOutputStream("message.dat")); + out.write(buff); + out.close(); + + Runtime rt = Runtime.getRuntime(); + rt.exec(new String[] { + "gsmsend", "-s", "emp01.baidu.com:15009", + "13811569457@$(cat message.dat)" }, + new String[] { "LANG=\"zh_CN.gbk\"" }); + } +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedMobileReporter.java b/src/java/hma/monitor/strategy/trigger/task/AttachedMobileReporter.java new file mode 100644 index 0000000..51a7d29 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedMobileReporter.java @@ -0,0 +1,151 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.util.EasyWebGet; + +import java.io.DataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.Charset; +import java.text.SimpleDateFormat; + +/** + * @author guoyezhi + * + */ +public class AttachedMobileReporter extends MonitorStrategyAttachedTask { + + private String[] servers = null; + + private String hmaGSMsend = null; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param taskWorker + * @param workerArgs + * @param taskArgs + */ + public AttachedMobileReporter( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments taskArgs) { + super(taskWorker, workerArgs, taskArgs); + this.servers = + MonitorManager.getGlobalConf().getStrings("baidu.gsm.servers"); + this.hmaGSMsend = + System.getProperty("hma.home.dir") + "/conf/" + + MonitorManager.getMonitoredClusterName() + "/gsmsend.hma"; + } + + private void doSendSM(String[] receivers, String message) { + + if(MonitorManager.isAlarm() == false){ + System.out.println("Oops..." + message + ", monitor is shutdown ..."); + return; + } + + Runtime rt = Runtime.getRuntime(); + + for (int i = 0; i < receivers.length; i++) { + for (int j = 0; j < servers.length; j++) { + try { + for (String str : new String[] { + "gsmsend", "-s", servers[j].trim(), + receivers[i].trim() + "@" + '"' + message + '"' }) { + System.out.print(str + " "); + } + System.out.println(); + + rt.exec(new String[] { + hmaGSMsend, + "-s", servers[j].trim(), + receivers[i].trim(), + message }); + + break; + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + + + + + + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + String[] receivers = getWorkerArgs(); + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + String infoLevel = taskArgs.getAlarmLevel(); + Configuration conf = MonitorManager.getGlobalConf(); + String alarmThres = conf.get( + "mobile.reporter.alarm.level.threshold", + "NOTICE"); + + if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + infoLevel, alarmThres) <= 0) { + return; + } + + StringBuilder msgBuilder = new StringBuilder(); + msgBuilder.append("[HMA]"); + msgBuilder.append("[" + taskArgs.getTargetSystemName() + "]"); + msgBuilder.append("[" + taskArgs.getAlarmLevel() + "]"); + msgBuilder.append("[" + taskArgs.getMonitorStrategyName() + "]"); + msgBuilder.append("[" + taskArgs.getKeyInfo() + "]"); + msgBuilder.append("[" + dateFormat.format(taskArgs.getTimestamp()) + "]"); + + doSendSM(receivers, msgBuilder.toString()); + + /* + * add by yangsen01 + * for Hermes. important alerm should call + */ + String url = "http://jingle.baidu.com/EIPCommunicator/pushIncident"; + String params="alertJson={"+ + "\"service\":\"dpf\","+ + "\"token\":\"551256bc833bd8d80f3c9869\","+ + "\"smsReceiver\":\"yangsen01\","+ + "\"details\":\""+ msgBuilder.toString() +"\","+ + "\"nodeId\":\"100030575\","+ + "\"nodeName\":\"BAIDU_INF_HADOOP\""+ + "}"; + EasyWebGet.post(url, params); + } + + public static void main(String[] args) throws IOException { + + String message = "涓枃鐭俊娴嬭瘯"; + byte[] buff = + Charset.forName("gbk").encode(message).array(); + message = ""; + DataOutputStream out = new DataOutputStream( + new FileOutputStream("message.dat")); + out.write(buff); + out.close(); + + Runtime rt = Runtime.getRuntime(); + rt.exec(new String[] { + "gsmsend", "-s", "emp01.baidu.com:15009", + "13811569457@$(cat message.dat)" }, + new String[] { "LANG=\"zh_CN.gbk\"" }); + } +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.bak b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.bak new file mode 100644 index 0000000..54651ce --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.bak @@ -0,0 +1,215 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.util.HttpClientHelper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.http.client.ClientProtocolException; + +/** + * @author guoyezhi + * + */ +public class AttachedNoahEIPReporter extends MonitorStrategyAttachedTask { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private Map noahNodeIDMap = null; + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedNoahEIPReporter( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + this.noahNodeIDMap = initNoahNodeIDMap(); + } + + private Map initNoahNodeIDMap() { + noahNodeIDMap = new HashMap(); + + // TODO: update automatically in future + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_ST-OFF", 13196); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_ECOM-OFF", 13188); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_ST-ON", 13191); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_ECOM-ON", 13185); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_CH-BACKUP", 15815); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_JP", 13350); + + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_SZWG-CH", 23863); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_WG-STON", 20709); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_WG-STOFF", 20941); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_WG-ECOMON", 20703); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_WG-ECOMRT", 21745); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_SZWG-KUN", 30785); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_SZWG-TAISHAN",26933); + noahNodeIDMap.put("BAIDU_OP_dmop_HADOOP_HY-ECOMOFF", 20934); + + return noahNodeIDMap; + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + Configuration conf = MonitorManager.getGlobalConf(); + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + + String alarmLevel = taskArgs.getAlarmLevel(); + if (alarmLevel.equals("INFO")) { + // Report of `Info' level should not be sent! + return; + } + + // translate HMA alarm level into EIP urgency + Integer urgency = translateAlarmLevelIntoEIPUrgency(alarmLevel); + + /* + * 1：IFC + * 2: Mizar + * 3: Distributed Monitor + */ + Integer origin = 3; + + String clusterName = taskArgs.getTargetSystemName(); + + String nodePath = getNoahNodePath(clusterName, conf); + Integer nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + nodePath = getNoahNodePath2(clusterName, conf); + nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + throw new RuntimeException( + "Cluster \"" + clusterName + "\" is not a Noah Service Node!"); + } + } + + String title = + "[HMA]" + + "[" + clusterName.toUpperCase() + "]" + + "[" + taskArgs.getMonitorStrategyName() + "]" + + "[" + taskArgs.getKeyInfo() + "]"; + + String ruleName = taskArgs.getMonitorStrategyName(); + Integer ruleId = ruleName.hashCode(); + + String[] receivers = getWorkerArgs(); + String receiverList = ""; + for (String receiver : receivers) { + receiverList = receiverList + receiver + ";"; + } + + String alarmedAt = dateFormat.format(taskArgs.getTimestamp()); + String hostname = "hma.dmop.baidu.com"; + + String info = taskArgs.getFullInfo(); + + // setup url + String url = conf.get( + "noah.eip.url", + "http://noah.baidu.com/eip/event/extImportor.do"); + Map params = new HashMap(); + params.put("origin", origin.toString()); + params.put("title", title); + params.put("ruleId", ruleId.toString()); + params.put("ruleName", ruleName); + params.put("nodeId", nodeId.toString()); + params.put("nodeName", nodePath); + params.put("alarmedAt", alarmedAt); + params.put("urgency", urgency.toString()); + params.put("receiver", receiverList); + params.put("info", info); + params.put("hostname", hostname); + + // send report + String rspContent = null; + try { + rspContent = HttpClientHelper.postRequest(url, params, "utf-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } catch (ClientProtocolException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + if (rspContent != null) { + System.out.println(rspContent); + } + + } + + private String getNoahNodePath(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + if (nodeName.startsWith("SZWG-")) { + nodeName = nodeName.replaceFirst("SZWG-", "WG-"); + } + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_OP_dmop_HADOOP_") + nodeName; + return nodePath; + } + + private String getNoahNodePath2(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_OP_dmop_HADOOP_") + nodeName; + return nodePath; + } + + private int translateAlarmLevelIntoEIPUrgency(String alarmLevel) { + int urgency = 0; + if (alarmLevel.equals("EMERG") + || alarmLevel.equals("CRITICAL")) { + urgency = 1; + } else if (alarmLevel.equals("WARN") + || alarmLevel.equals("WARNING")) { + urgency = 2; + } else if (alarmLevel.equals("NOTICE")) { + urgency = 3; + } else if (alarmLevel.equals("INFO")) { + urgency = 4; + } else { + urgency = 5; + } + return urgency; + } + + + /** + * for unit testing + * + * @param args + * @throws InterruptedException + */ + public static void main(String[] args) throws InterruptedException { + String[] workerArgs = + new String[]{ "g_dmop_hadoop_lungang" }; + MonitorStrategyAttachedTaskArguments taskArgs = + new MonitorStrategyAttachedTaskArguments( + "SZWG-STON", "Test Strategy", "NOTICE", "Test Key Info", + "Test Full Info", new Date()); + new AttachedNoahEIPReporter(null, workerArgs, taskArgs).run(); + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java new file mode 100644 index 0000000..2b382ba --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java @@ -0,0 +1,227 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.util.HttpClientHelper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.http.client.ClientProtocolException; + +/** + * @author guoyezhi + * + */ +public class AttachedNoahEIPReporter extends MonitorStrategyAttachedTask { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private Map noahNodeIDMap = null; + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedNoahEIPReporter( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + this.noahNodeIDMap = initNoahNodeIDMap(); + } + + private Map initNoahNodeIDMap() { + noahNodeIDMap = new HashMap(); + + // TODO: update automatically in future + noahNodeIDMap.put("BAIDU_INF_HADOOP_JP", 13350); + + + // http://tc-oped-dev03.tc.baidu.com:8110/service-tree/v1/node/path_BAIDU_INF_HADOOP_SZWG-ECOMOFF + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-CH", 23863); + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-STON", 20709); + + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-STOFF", 20704); + + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-ECOMON", 20703); + + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-ECOMRT", 21745); + + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-KUN", 30785); + + noahNodeIDMap.put("BAIDU_INF_HADOOP_HY-ECOMOFF", 20934); + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-ECOMOFF", 37407); + + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-RANK", 32026); + + noahNodeIDMap.put("BAIDU_INF_HADOOP_NJ01-YULONG", 200002459); + + noahNodeIDMap.put("BAIDU_INF_HADOOP_NJ01-NANLING", 200002291); + + + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-DAYU", 200003292); + + return noahNodeIDMap; + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + Configuration conf = MonitorManager.getGlobalConf(); + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + + String alarmLevel = taskArgs.getAlarmLevel(); + if (alarmLevel.equals("INFO")) { + // Report of `Info' level should not be sent! + return; + } + + // translate HMA alarm level into EIP urgency + Integer urgency = translateAlarmLevelIntoEIPUrgency(alarmLevel); + + /* + * 1：IFC + * 2: Mizar + * 3: Distributed Monitor + */ + Integer origin = 3; + + String clusterName = taskArgs.getTargetSystemName(); + + String nodePath = getNoahNodePath(clusterName, conf); + Integer nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + nodePath = getNoahNodePath2(clusterName, conf); + nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + throw new RuntimeException( + "Cluster \"" + clusterName + "\" is not a Noah Service Node!"); + } + } + + String title = + "[HMA]" + + "[" + clusterName.toUpperCase() + "]" + + "[" + taskArgs.getMonitorStrategyName() + "]" + + "[" + taskArgs.getKeyInfo() + "]"; + + String ruleName = taskArgs.getMonitorStrategyName(); + Integer ruleId = ruleName.hashCode(); + + String[] receivers = getWorkerArgs(); + String receiverList = ""; + for (String receiver : receivers) { + receiverList = receiverList + receiver + ";"; + } + + String alarmedAt = dateFormat.format(taskArgs.getTimestamp()); + String hostname = "hma.dmop.baidu.com"; + + String info = taskArgs.getFullInfo(); + + // setup url + String url = conf.get( + "noah.eip.url", + "http://noah.baidu.com/eip/event/extImportor.do"); + Map params = new HashMap(); + params.put("origin", origin.toString()); + params.put("title", title); + params.put("ruleId", ruleId.toString()); + params.put("ruleName", ruleName); + params.put("nodeId", nodeId.toString()); + params.put("nodeName", nodePath); + params.put("alarmedAt", alarmedAt); + params.put("urgency", urgency.toString()); + params.put("receiver", receiverList); + params.put("info", info); + params.put("hostname", hostname); + + // send report + String rspContent = null; + try { + rspContent = HttpClientHelper.postRequest(url, params, "utf-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } catch (ClientProtocolException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + if (rspContent != null) { + System.out.println(rspContent); + } + + } + + private String getNoahNodePath(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + if (nodeName.startsWith("WG-")) { + nodeName = nodeName.replaceFirst("WG-", "SZWG-"); + } + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_INF_HADOOP_") + nodeName; + return nodePath; + } + + private String getNoahNodePath2(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_INF_DPF_HADOOP_") + nodeName; + return nodePath; + } + + private int translateAlarmLevelIntoEIPUrgency(String alarmLevel) { + int urgency = 0; + if (alarmLevel.equals("EMERG") + || alarmLevel.equals("CRITICAL")) { + urgency = 1; + } else if (alarmLevel.equals("WARN") + || alarmLevel.equals("WARNING")) { + urgency = 2; + } else if (alarmLevel.equals("NOTICE")) { + urgency = 3; + } else if (alarmLevel.equals("INFO")) { + urgency = 4; + } else { + urgency = 5; + } + return urgency; + } + + + /** + * for unit testing + * + * @param args + * @throws InterruptedException + */ + public static void main(String[] args) throws InterruptedException { + String[] workerArgs = + new String[]{ "g_dmop_hadoop_lungang" }; + MonitorStrategyAttachedTaskArguments taskArgs = + new MonitorStrategyAttachedTaskArguments( + "SZWG-STON", "Test Strategy", "NOTICE", "Test Key Info", + "Test Full Info", new Date()); + new AttachedNoahEIPReporter(null, workerArgs, taskArgs).run(); + } + +} + diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java-20120702 b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java-20120702 new file mode 100644 index 0000000..66c848b --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java-20120702 @@ -0,0 +1,215 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.util.HttpClientHelper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.http.client.ClientProtocolException; + +/** + * @author guoyezhi + * + */ +public class AttachedNoahEIPReporter extends MonitorStrategyAttachedTask { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private Map noahNodeIDMap = null; + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedNoahEIPReporter( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + this.noahNodeIDMap = initNoahNodeIDMap(); + } + + private Map initNoahNodeIDMap() { + noahNodeIDMap = new HashMap(); + + // TODO: update automatically in future + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_ST-OFF", 13196); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_ECOM-OFF", 13188); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_ST-ON", 13191); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_ECOM-ON", 13185); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_CH-BACKUP", 15815); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_JP", 13350); + + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_SZWG-CH", 23863); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_WG-STON", 20709); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_WG-STOFF", 20704); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_WG-ECOMON", 20703); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_WG-ECOMRT", 21745); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_SZWG-KUN", 30785); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_SZWG-TAISHAN",26933); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_HY-ECOMOFF", 20934); + + return noahNodeIDMap; + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + Configuration conf = MonitorManager.getGlobalConf(); + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + + String alarmLevel = taskArgs.getAlarmLevel(); + if (alarmLevel.equals("INFO")) { + // Report of `Info' level should not be sent! + return; + } + + // translate HMA alarm level into EIP urgency + Integer urgency = translateAlarmLevelIntoEIPUrgency(alarmLevel); + + /* + * 1：IFC + * 2: Mizar + * 3: Distributed Monitor + */ + Integer origin = 3; + + String clusterName = taskArgs.getTargetSystemName(); + + String nodePath = getNoahNodePath(clusterName, conf); + Integer nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + nodePath = getNoahNodePath2(clusterName, conf); + nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + throw new RuntimeException( + "Cluster \"" + clusterName + "\" is not a Noah Service Node!"); + } + } + + String title = + "[HMA]" + + "[" + clusterName.toUpperCase() + "]" + + "[" + taskArgs.getMonitorStrategyName() + "]" + + "[" + taskArgs.getKeyInfo() + "]"; + + String ruleName = taskArgs.getMonitorStrategyName(); + Integer ruleId = ruleName.hashCode(); + + String[] receivers = getWorkerArgs(); + String receiverList = ""; + for (String receiver : receivers) { + receiverList = receiverList + receiver + ";"; + } + + String alarmedAt = dateFormat.format(taskArgs.getTimestamp()); + String hostname = "hma.dmop.baidu.com"; + + String info = taskArgs.getFullInfo(); + + // setup url + String url = conf.get( + "noah.eip.url", + "http://noah.baidu.com/eip/event/extImportor.do"); + Map params = new HashMap(); + params.put("origin", origin.toString()); + params.put("title", title); + params.put("ruleId", ruleId.toString()); + params.put("ruleName", ruleName); + params.put("nodeId", nodeId.toString()); + params.put("nodeName", nodePath); + params.put("alarmedAt", alarmedAt); + params.put("urgency", urgency.toString()); + params.put("receiver", receiverList); + params.put("info", info); + params.put("hostname", hostname); + + // send report + String rspContent = null; + try { + rspContent = HttpClientHelper.postRequest(url, params, "utf-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } catch (ClientProtocolException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + if (rspContent != null) { + System.out.println(rspContent); + } + + } + + private String getNoahNodePath(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + if (nodeName.startsWith("SZWG-")) { + nodeName = nodeName.replaceFirst("SZWG-", "WG-"); + } + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_OP_INF-OP_dpf-op_HADOOP_") + nodeName; + return nodePath; + } + + private String getNoahNodePath2(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_OP_INF-OP_dpf-op_HADOOP_") + nodeName; + return nodePath; + } + + private int translateAlarmLevelIntoEIPUrgency(String alarmLevel) { + int urgency = 0; + if (alarmLevel.equals("EMERG") + || alarmLevel.equals("CRITICAL")) { + urgency = 1; + } else if (alarmLevel.equals("WARN") + || alarmLevel.equals("WARNING")) { + urgency = 2; + } else if (alarmLevel.equals("NOTICE")) { + urgency = 3; + } else if (alarmLevel.equals("INFO")) { + urgency = 4; + } else { + urgency = 5; + } + return urgency; + } + + + /** + * for unit testing + * + * @param args + * @throws InterruptedException + */ + public static void main(String[] args) throws InterruptedException { + String[] workerArgs = + new String[]{ "g_dmop_hadoop_lungang" }; + MonitorStrategyAttachedTaskArguments taskArgs = + new MonitorStrategyAttachedTaskArguments( + "SZWG-STON", "Test Strategy", "NOTICE", "Test Key Info", + "Test Full Info", new Date()); + new AttachedNoahEIPReporter(null, workerArgs, taskArgs).run(); + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java.20130114 b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java.20130114 new file mode 100644 index 0000000..7a17cba --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java.20130114 @@ -0,0 +1,215 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.util.HttpClientHelper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.http.client.ClientProtocolException; + +/** + * @author guoyezhi + * + */ +public class AttachedNoahEIPReporter extends MonitorStrategyAttachedTask { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private Map noahNodeIDMap = null; + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedNoahEIPReporter( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + this.noahNodeIDMap = initNoahNodeIDMap(); + } + + private Map initNoahNodeIDMap() { + noahNodeIDMap = new HashMap(); + + // TODO: update automatically in future + noahNodeIDMap.put("BAIDU_INF_HADOOP_ST-OFF", 13196); + noahNodeIDMap.put("BAIDU_INF_HADOOP_ECOM-OFF", 13188); + noahNodeIDMap.put("BAIDU_INF_HADOOP_ST-ON", 13191); + noahNodeIDMap.put("BAIDU_INF_HADOOP_ECOM-ON", 13185); + noahNodeIDMap.put("BAIDU_INF_HADOOP_CH-BACKUP", 15815); + noahNodeIDMap.put("BAIDU_INF_HADOOP_JP", 13350); + + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-CH", 23863); + noahNodeIDMap.put("BAIDU_INF_HADOOP_WG-STON", 20709); + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-STOFF", 20704); + noahNodeIDMap.put("BAIDU_INF_HADOOP_WG-ECOMON", 20703); + noahNodeIDMap.put("BAIDU_INF_HADOOP_WG-ECOMRT", 21745); + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-KUN", 30785); + noahNodeIDMap.put("BAIDU_INF_HADOOP_SZWG-TAISHAN",26933); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HY-ECOMOFF", 20934); + + return noahNodeIDMap; + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + Configuration conf = MonitorManager.getGlobalConf(); + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + + String alarmLevel = taskArgs.getAlarmLevel(); + if (alarmLevel.equals("INFO")) { + // Report of `Info' level should not be sent! + return; + } + + // translate HMA alarm level into EIP urgency + Integer urgency = translateAlarmLevelIntoEIPUrgency(alarmLevel); + + /* + * 1：IFC + * 2: Mizar + * 3: Distributed Monitor + */ + Integer origin = 3; + + String clusterName = taskArgs.getTargetSystemName(); + + String nodePath = getNoahNodePath(clusterName, conf); + Integer nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + nodePath = getNoahNodePath2(clusterName, conf); + nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + throw new RuntimeException( + "Cluster \"" + clusterName + "\" is not a Noah Service Node!"); + } + } + + String title = + "[HMA]" + + "[" + clusterName.toUpperCase() + "]" + + "[" + taskArgs.getMonitorStrategyName() + "]" + + "[" + taskArgs.getKeyInfo() + "]"; + + String ruleName = taskArgs.getMonitorStrategyName(); + Integer ruleId = ruleName.hashCode(); + + String[] receivers = getWorkerArgs(); + String receiverList = ""; + for (String receiver : receivers) { + receiverList = receiverList + receiver + ";"; + } + + String alarmedAt = dateFormat.format(taskArgs.getTimestamp()); + String hostname = "hma.dmop.baidu.com"; + + String info = taskArgs.getFullInfo(); + + // setup url + String url = conf.get( + "noah.eip.url", + "http://noah.baidu.com/eip/event/extImportor.do"); + Map params = new HashMap(); + params.put("origin", origin.toString()); + params.put("title", title); + params.put("ruleId", ruleId.toString()); + params.put("ruleName", ruleName); + params.put("nodeId", nodeId.toString()); + params.put("nodeName", nodePath); + params.put("alarmedAt", alarmedAt); + params.put("urgency", urgency.toString()); + params.put("receiver", receiverList); + params.put("info", info); + params.put("hostname", hostname); + + // send report + String rspContent = null; + try { + rspContent = HttpClientHelper.postRequest(url, params, "utf-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } catch (ClientProtocolException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + if (rspContent != null) { + System.out.println(rspContent); + } + + } + + private String getNoahNodePath(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + if (nodeName.startsWith("SZWG-")) { + nodeName = nodeName.replaceFirst("SZWG-", "WG-"); + } + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_INF_DPF_HADOOP_") + nodeName; + return nodePath; + } + + private String getNoahNodePath2(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_INF_DPF_HADOOP_") + nodeName; + return nodePath; + } + + private int translateAlarmLevelIntoEIPUrgency(String alarmLevel) { + int urgency = 0; + if (alarmLevel.equals("EMERG") + || alarmLevel.equals("CRITICAL")) { + urgency = 1; + } else if (alarmLevel.equals("WARN") + || alarmLevel.equals("WARNING")) { + urgency = 2; + } else if (alarmLevel.equals("NOTICE")) { + urgency = 3; + } else if (alarmLevel.equals("INFO")) { + urgency = 4; + } else { + urgency = 5; + } + return urgency; + } + + + /** + * for unit testing + * + * @param args + * @throws InterruptedException + */ + public static void main(String[] args) throws InterruptedException { + String[] workerArgs = + new String[]{ "g_dmop_hadoop_lungang" }; + MonitorStrategyAttachedTaskArguments taskArgs = + new MonitorStrategyAttachedTaskArguments( + "SZWG-STON", "Test Strategy", "NOTICE", "Test Key Info", + "Test Full Info", new Date()); + new AttachedNoahEIPReporter(null, workerArgs, taskArgs).run(); + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java_20120822_new b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java_20120822_new new file mode 100644 index 0000000..50faf3c --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java_20120822_new @@ -0,0 +1,214 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.util.HttpClientHelper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.http.client.ClientProtocolException; + +/** + * @author guoyezhi + * + */ +public class AttachedNoahEIPReporter extends MonitorStrategyAttachedTask { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private Map noahNodeIDMap = null; + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedNoahEIPReporter( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + this.noahNodeIDMap = initNoahNodeIDMap(); + } + + private Map initNoahNodeIDMap() { + noahNodeIDMap = new HashMap(); + + // TODO: update automatically in future + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_ST-OFF", 13196); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_ECOM-OFF", 13188); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_ST-ON", 13191); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_ECOM-ON", 13185); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_CH-BACKUP", 15815); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_JP", 13350); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_SZWG-CH", 23863); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_WG-STON", 20709); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_SZWG-STOFF", 20704); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_WG-ECOMON", 20703); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_WG-ECOMRT", 21745); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_SZWG-KUN", 30785); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_SZWG-TAISHAN", 26933); + noahNodeIDMap.put("BAIDU_INF_HADOOP_HADOOP_HY-ECOMOFF", 20934); + + return noahNodeIDMap; + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + Configuration conf = MonitorManager.getGlobalConf(); + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + + String alarmLevel = taskArgs.getAlarmLevel(); + if (alarmLevel.equals("INFO")) { + // Report of `Info' level should not be sent! + return; + } + + // translate HMA alarm level into EIP urgency + Integer urgency = translateAlarmLevelIntoEIPUrgency(alarmLevel); + + /* + * 1：IFC + * 2: Mizar + * 3: Distributed Monitor + */ + Integer origin = 3; + + String clusterName = taskArgs.getTargetSystemName(); + + String nodePath = getNoahNodePath(clusterName, conf); + Integer nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + nodePath = getNoahNodePath2(clusterName, conf); + nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + throw new RuntimeException( + "Cluster \"" + clusterName + "\" is not a Noah Service Node!"); + } + } + + String title = + "[HMA]" + + "[" + clusterName.toUpperCase() + "]" + + "[" + taskArgs.getMonitorStrategyName() + "]" + + "[" + taskArgs.getKeyInfo() + "]"; + + String ruleName = taskArgs.getMonitorStrategyName(); + Integer ruleId = ruleName.hashCode(); + + String[] receivers = getWorkerArgs(); + String receiverList = ""; + for (String receiver : receivers) { + receiverList = receiverList + receiver + ";"; + } + + String alarmedAt = dateFormat.format(taskArgs.getTimestamp()); + String hostname = "hma.dmop.baidu.com"; + + String info = taskArgs.getFullInfo(); + + // setup url + String url = conf.get( + "noah.eip.url", + "http://noah.baidu.com/eip/event/extImportor.do"); + Map params = new HashMap(); + params.put("origin", origin.toString()); + params.put("title", title); + params.put("ruleId", ruleId.toString()); + params.put("ruleName", ruleName); + params.put("nodeId", nodeId.toString()); + params.put("nodeName", nodePath); + params.put("alarmedAt", alarmedAt); + params.put("urgency", urgency.toString()); + params.put("receiver", receiverList); + params.put("info", info); + params.put("hostname", hostname); + + // send report + String rspContent = null; + try { + rspContent = HttpClientHelper.postRequest(url, params, "utf-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } catch (ClientProtocolException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + if (rspContent != null) { + System.out.println(rspContent); + } + + } + + private String getNoahNodePath(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + if (nodeName.startsWith("SZWG-")) { + nodeName = nodeName.replaceFirst("SZWG-", "WG-"); + } + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_INF_DPF_HADOOP_") + nodeName; + return nodePath; + } + + private String getNoahNodePath2(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_INF_DPF_HADOOP_") + nodeName; + return nodePath; + } + + private int translateAlarmLevelIntoEIPUrgency(String alarmLevel) { + int urgency = 0; + if (alarmLevel.equals("EMERG") + || alarmLevel.equals("CRITICAL")) { + urgency = 1; + } else if (alarmLevel.equals("WARN") + || alarmLevel.equals("WARNING")) { + urgency = 2; + } else if (alarmLevel.equals("NOTICE")) { + urgency = 3; + } else if (alarmLevel.equals("INFO")) { + urgency = 4; + } else { + urgency = 5; + } + return urgency; + } + + + /** + * for unit testing + * + * @param args + * @throws InterruptedException + */ + public static void main(String[] args) throws InterruptedException { + String[] workerArgs = + new String[]{ "g_dmop_hadoop_lungang" }; + MonitorStrategyAttachedTaskArguments taskArgs = + new MonitorStrategyAttachedTaskArguments( + "SZWG-STON", "Test Strategy", "NOTICE", "Test Key Info", + "Test Full Info", new Date()); + new AttachedNoahEIPReporter(null, workerArgs, taskArgs).run(); + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java_20120823 b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java_20120823 new file mode 100644 index 0000000..7630f0f --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedNoahEIPReporter.java_20120823 @@ -0,0 +1,215 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.util.HttpClientHelper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.http.client.ClientProtocolException; + +/** + * @author guoyezhi + * + */ +public class AttachedNoahEIPReporter extends MonitorStrategyAttachedTask { + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private Map noahNodeIDMap = null; + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedNoahEIPReporter( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + this.noahNodeIDMap = initNoahNodeIDMap(); + } + + private Map initNoahNodeIDMap() { + noahNodeIDMap = new HashMap(); + + // TODO: update automatically in future + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_ST-OFF", 13196); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_ECOM-OFF", 13188); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_ST-ON", 13191); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_ECOM-ON", 13185); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_CH-BACKUP", 15815); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_JP", 13350); + + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_SZWG-CH", 23863); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_WG-STON", 20709); + noahNodeIDMap.put("BAIDU_INF_DPF_HADOOP_SZWG-STOFF", 20704); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_WG-ECOMON", 20703); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_WG-ECOMRT", 21745); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_SZWG-KUN", 30785); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_SZWG-TAISHAN",26933); + noahNodeIDMap.put("BAIDU_OP_INF-OP_dpf-op_HADOOP_HY-ECOMOFF", 20934); + + return noahNodeIDMap; + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + Configuration conf = MonitorManager.getGlobalConf(); + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + + String alarmLevel = taskArgs.getAlarmLevel(); + if (alarmLevel.equals("INFO")) { + // Report of `Info' level should not be sent! + return; + } + + // translate HMA alarm level into EIP urgency + Integer urgency = translateAlarmLevelIntoEIPUrgency(alarmLevel); + + /* + * 1：IFC + * 2: Mizar + * 3: Distributed Monitor + */ + Integer origin = 3; + + String clusterName = taskArgs.getTargetSystemName(); + + String nodePath = getNoahNodePath(clusterName, conf); + Integer nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + nodePath = getNoahNodePath2(clusterName, conf); + nodeId = noahNodeIDMap.get(nodePath); + if (nodeId == null) { + throw new RuntimeException( + "Cluster \"" + clusterName + "\" is not a Noah Service Node!"); + } + } + + String title = + "[HMA]" + + "[" + clusterName.toUpperCase() + "]" + + "[" + taskArgs.getMonitorStrategyName() + "]" + + "[" + taskArgs.getKeyInfo() + "]"; + + String ruleName = taskArgs.getMonitorStrategyName(); + Integer ruleId = ruleName.hashCode(); + + String[] receivers = getWorkerArgs(); + String receiverList = ""; + for (String receiver : receivers) { + receiverList = receiverList + receiver + ";"; + } + + String alarmedAt = dateFormat.format(taskArgs.getTimestamp()); + String hostname = "hma.dmop.baidu.com"; + + String info = taskArgs.getFullInfo(); + + // setup url + String url = conf.get( + "noah.eip.url", + "http://noah.baidu.com/eip/event/extImportor.do"); + Map params = new HashMap(); + params.put("origin", origin.toString()); + params.put("title", title); + params.put("ruleId", ruleId.toString()); + params.put("ruleName", ruleName); + params.put("nodeId", nodeId.toString()); + params.put("nodeName", nodePath); + params.put("alarmedAt", alarmedAt); + params.put("urgency", urgency.toString()); + params.put("receiver", receiverList); + params.put("info", info); + params.put("hostname", hostname); + + // send report + String rspContent = null; + try { + rspContent = HttpClientHelper.postRequest(url, params, "utf-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } catch (ClientProtocolException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + if (rspContent != null) { + System.out.println(rspContent); + } + + } + + private String getNoahNodePath(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + if (nodeName.startsWith("SZWG-")) { + nodeName = nodeName.replaceFirst("SZWG-", "WG-"); + } + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_INF_DPF_HADOOP_") + nodeName; + return nodePath; + } + + private String getNoahNodePath2(String clusterName, Configuration conf) { + String nodeName = clusterName.toUpperCase(); + String nodePath = conf.get( + "noah.dmop.hadoop.service.node.prefix", + "BAIDU_INF_DPF_HADOOP_") + nodeName; + return nodePath; + } + + private int translateAlarmLevelIntoEIPUrgency(String alarmLevel) { + int urgency = 0; + if (alarmLevel.equals("EMERG") + || alarmLevel.equals("CRITICAL")) { + urgency = 1; + } else if (alarmLevel.equals("WARN") + || alarmLevel.equals("WARNING")) { + urgency = 2; + } else if (alarmLevel.equals("NOTICE")) { + urgency = 3; + } else if (alarmLevel.equals("INFO")) { + urgency = 4; + } else { + urgency = 5; + } + return urgency; + } + + + /** + * for unit testing + * + * @param args + * @throws InterruptedException + */ + public static void main(String[] args) throws InterruptedException { + String[] workerArgs = + new String[]{ "g_dmop_hadoop_lungang" }; + MonitorStrategyAttachedTaskArguments taskArgs = + new MonitorStrategyAttachedTaskArguments( + "SZWG-STON", "Test Strategy", "NOTICE", "Test Key Info", + "Test Full Info", new Date()); + new AttachedNoahEIPReporter(null, workerArgs, taskArgs).run(); + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedSREMobileReporter.java b/src/java/hma/monitor/strategy/trigger/task/AttachedSREMobileReporter.java new file mode 100644 index 0000000..f2b8521 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedSREMobileReporter.java @@ -0,0 +1,147 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.web.HMAJspHelper; + +import java.text.SimpleDateFormat; + +/** + * @author guoyezhi + * + */ +public class AttachedSREMobileReporter extends MonitorStrategyAttachedTask { + + private String[] servers = null; + + private String hmaGSMsend = null; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedSREMobileReporter(String taskWorker, String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + this.servers = + MonitorManager.getGlobalConf().getStrings("baidu.gsm.servers"); + this.hmaGSMsend = + System.getProperty("hma.home.dir") + "/conf/" + + MonitorManager.getMonitoredClusterName() + "/gsmsend.hma"; + } + + private void doSendSM(String[] receivers, String message) { + + Runtime rt = Runtime.getRuntime(); + + for (int i = 0; i < receivers.length; i++) { + for (int j = 0; j < servers.length; j++) { + try { + for (String str : new String[] { + "gsmsend", "-s", servers[j].trim(), + receivers[i].trim() + "@" + '"' + message + '"' }) { + System.out.print(str + " "); + } + System.out.println(); + + String [] smscommand = new String[] { + hmaGSMsend, + "-s", servers[j].trim(), + receivers[i].trim(), + message }; + rt.exec(smscommand); + for(String str:smscommand){ + + System.out.println("kain's sms log "+ str ); + } + + + break; + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + String monitorStrategyName = + taskArgs.getMonitorStrategyName(); + String alarmLevel = taskArgs.getAlarmLevel(); + + Configuration conf = MonitorManager.getGlobalConf(); + String alarmThres = conf.get( + "mobile.reporter.alarm.level.threshold", + "NOTICE"); + + if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + alarmLevel, alarmThres) <= 0) { + return; + } + + String alarmTag = getAlarmTag(monitorStrategyName, alarmLevel); + String[] receivers = getReceivers(alarmLevel, conf); + + StringBuilder msgBuilder = new StringBuilder(); + msgBuilder.append("[HMA]"); + msgBuilder.append("[" + taskArgs.getTargetSystemName() + "]"); + msgBuilder.append("[" + alarmTag + "]"); + msgBuilder.append("[" + HMAJspHelper.getMonitorStrategyAlias(monitorStrategyName) + "]"); + msgBuilder.append("[" + taskArgs.getKeyInfo() + "]"); + msgBuilder.append("[" + dateFormat.format(taskArgs.getTimestamp()) + "]"); + + doSendSM(receivers, msgBuilder.toString()); + + } + + private String getAlarmTag(String monitorStrategyName, String alarmLevel) { + String alarmTag = null; + if (monitorStrategyName.equals("INode Quota") || + monitorStrategyName.equals("Space Quota")) { + alarmTag = alarmLevel; + } else { + if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + alarmLevel, "WARN") == 0) { + alarmTag = "WARN"; + } else if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + alarmLevel, "CRITICAL") >= 0) { + alarmTag = "CRITICAL"; + } else { + alarmTag = "CRITICAL"; + } + } + return alarmTag; + } + + private String[] getReceivers(String infoLevel, Configuration conf) { + String[] receivers = null; + if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + infoLevel, "WARN") == 0) { + receivers = conf.getStrings( + "sre.mobile.reporter.warning.level.alarm.receivers"); + } else if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + infoLevel, "CRITICAL") >= 0) { + receivers = conf.getStrings( + "sre.mobile.reporter.critical.level.alarm.receivers"); + } else { + receivers = getWorkerArgs(); + } + return receivers; + } + +} + diff --git a/src/java/hma/monitor/strategy/trigger/task/AttachedSREMobileReporter.java.bak.yangsenVersion b/src/java/hma/monitor/strategy/trigger/task/AttachedSREMobileReporter.java.bak.yangsenVersion new file mode 100644 index 0000000..9174495 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/AttachedSREMobileReporter.java.bak.yangsenVersion @@ -0,0 +1,293 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; +import hma.util.EasyWebGet; +import hma.web.HMAJspHelper; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * @author guoyezhi + */ +public class AttachedSREMobileReporter extends MonitorStrategyAttachedTask { + + private String[] servers = null; + + private String hmaGSMsend = null; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + public static final Log LOG = LogFactory + .getLog(AttachedSREMobileReporter.class); + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public AttachedSREMobileReporter(String taskWorker, String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + super(taskWorker, workerArgs, args); + this.servers = + MonitorManager.getGlobalConf().getStrings("baidu.gsm.servers"); + this.hmaGSMsend = + System.getProperty("hma.home.dir") + "/conf/" + + MonitorManager.getMonitoredClusterName() + "/gsmsend.hma"; + } + + private void doSendTg ( String [ ] receivers, String message,String channel) { + // test receiverlist + String receiverlist="" ; + for (int i =0; i < receivers.length; i++){ + receiverlist = receiverlist +";"+receivers[i].trim(); + } + // receiverlist="masuhua;hadoop-mon@baidu.com"; + //receiverlist="masuhua;dpf-op@baidu.com;#z_dpf_mon;z_60613"; + message = message+"[Tg]"; + SimpleDateFormat dotime = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + try{ + String url = "http://jingle.baidu.com/alert/push"; + String params="{"+ + "\"appId\":\"620\","+ + "\"token\":\"551256bc833bd8d80f3c9869\","+ + "\"alertList\":{"+ "\"receiver\":\""+ receiverlist +"\","+ "\"channel\":\""+ channel +"\","+ "\"title\":\"HMA报警\","+ "\"description\":\""+ message +"\""+ "}"+ + "}"; + EasyWebGet.post(url, params); + System.out.println("[INFO ] http://jingle.baidu.com/alert/push done ! "+ params + dotime); + }catch(Exception e){ + + System.out.println("[ERROR ] http://jingle.baidu.com/alert/push error ! " +dotime); + e.printStackTrace(); + } + + } + + private void doSendSM(String[] receivers, String message) { + + Runtime rt = Runtime.getRuntime(); + + if (MonitorManager.isAlarm() == false) { + System.out.println("Oops..." + message + ", monitor is shutdown ..."); + return; + } + + for (int i = 0; i < receivers.length; i++) { + + String[] gsmsendComm = new String[3 + 2 * servers.length]; + gsmsendComm[0] = "gsmsend"; + for (int j = 0; j < servers.length; j++) { + gsmsendComm[1 + 2 * j] = "-s"; + gsmsendComm[2 + 2 * j] = servers[j].trim(); + } + gsmsendComm[1 + 2 * servers.length] = receivers[i].trim(); + gsmsendComm[2 + 2 * servers.length] = message; + + for (String str : gsmsendComm) { + System.out.print(str + " "); + } + System.out.println(); + + gsmsendComm[0] = hmaGSMsend; + try { + rt.exec(gsmsendComm); + LOG.info("[gsm] send [" + message + "] to " + receivers[i]); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + + } + } + + /* (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + + MonitorStrategyAttachedTaskArguments taskArgs = + getAttachedTaskArguments(); + String monitorStrategyName = + taskArgs.getMonitorStrategyName(); + String alarmLevel = taskArgs.getAlarmLevel(); + + Configuration conf = MonitorManager.getGlobalConf(); + String alarmThres = conf.get( + "mobile.reporter.alarm.level.threshold", + "NOTICE"); + + if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + alarmLevel, alarmThres) <= 0) { + return; + } + /**Begain add by chiwen01 temporary, will remove before 2015.1.1*/ + String Mon_name = HMAJspHelper.getMonitorStrategyAlias(monitorStrategyName); + SimpleDateFormat justHour = new SimpleDateFormat("HH"); + int forbiddenStartTime = 18; + int forbiddenEndTime = 10; + boolean overDay = false; + int thisTime = Integer.parseInt(justHour.format(new Date())); + if (forbiddenEndTime - forbiddenStartTime < 0) { + overDay = true; + } + if ((Mon_name.toLowerCase()).equals("unworkingdatanodes")) { + if (thisTime > forbiddenStartTime && thisTime < forbiddenEndTime && !overDay) { + System.out.println("[INFO] "+dateFormat.format(taskArgs.getTimestamp())+" The strategy: "+Mon_name+" forbiden > " + forbiddenStartTime + " && < "+forbiddenEndTime); + return; + } else if (!(thisTime < forbiddenStartTime && thisTime > forbiddenEndTime) && overDay) { + System.out.println("[INFO] "+dateFormat.format(taskArgs.getTimestamp())+" The strategy: "+Mon_name+" forbiden > " + forbiddenStartTime + " && < "+forbiddenEndTime); + return; + } + } + /**End add by chiwen01 temporary, will remove before 2015.1.1*/ + + String alarmTag = getAlarmTag(monitorStrategyName, alarmLevel); + String[] receivers = getReceivers(alarmLevel, conf); + + StringBuilder msgBuilder = new StringBuilder(); + msgBuilder.append("[HMA]"); + msgBuilder.append("[" + taskArgs.getTargetSystemName() + "]"); + msgBuilder.append("[" + alarmTag + "]"); + msgBuilder.append("[" + HMAJspHelper.getMonitorStrategyAlias(monitorStrategyName) + "]"); + msgBuilder.append("[" + taskArgs.getKeyInfo() + "]"); + msgBuilder.append("[" + dateFormat.format(taskArgs.getTimestamp()) + "]"); + + /* + * add by yangsen01 + * for check if the Strategy is blocked. + */ + if (false == checkStrategyBlock(taskArgs.getTargetSystemName(),monitorStrategyName)) + { + //gsmsend + doSendSM(receivers, msgBuilder.toString()); + doSendTg(receivers, msgBuilder.toString(),"email"); + + if ( taskArgs.getTargetSystemName().toString().equals("SZWG-STOFF") || taskArgs.getTargetSystemName().toString().equals("NJ01-YULONG") || taskArgs.getTargetSystemName().toString().equals("NMG01-MULAN") ){ + doSendTg(receivers, msgBuilder.toString(),"sms"); + System.out.println(" sendtg success ok !"); + } + + /* + * add by yangsen01 + * for Hermes. important alerm should call + */ + try{ + String url = "http://jingle.baidu.com/EIPCommunicator/pushIncident"; + String params="alertJson={"+ + "\"appId\":\"620\","+ + "\"token\":\"551256bc833bd8d80f3c9869\","+ + "\"smsReceiver\":\"yangsen01\","+ + "\"details\":\""+ msgBuilder.toString() +"\","+ + "\"nodeId\":\"100030575\","+ + "\"nodeName\":\"BAIDU_INF_HADOOP\""+ + "}"; + EasyWebGet.post(url, params); + System.out.println("[INFO ] EIPCommunicator pushIncident done ! "+ params + dateFormat.format(taskArgs.getTimestamp())); + }catch(Exception e){ + + System.out.println("[ERROR ] EIPCommunicator pushIncident error ! "+ dateFormat.format(taskArgs.getTimestamp())); + e.printStackTrace(); + } + } + else { + System.out.println("[INFO] "+dateFormat.format(taskArgs.getTimestamp())+" The strategy: "+monitorStrategyName+" of the cluster: "+taskArgs.getTargetSystemName()+"is blocked!"); + } + } + + /* + * add by yangsen01 + * for check if the Strategy is blocked. + */ + public boolean checkStrategyBlock(String cluster,String monitorStrategyName) { + + boolean isBlock=false; + try{ + + String url="jdbc:mysql://hma.db.dmop.baidu.com:3306/hma"; //JDBC connection string + Connection conn; + + conn = DriverManager.getConnection(url,"root","hmainit"); + Statement stmt = conn.createStatement(); + + String sql = "select shield_status from monitor_hadoop_shield where cluster_name='"+cluster+"' and strategy_name='"+monitorStrategyName+"';"; //get block status SQL + ResultSet rs = stmt.executeQuery(sql); + if (rs.next()) { + System.out.println(rs.getInt(1)); + if(1 == rs.getInt(1)) + isBlock=true; + } + rs.close(); + stmt.close(); + conn.close(); + + return isBlock; + }catch(Exception e) + { + System.out.println("[WARN] Check Strategy block status ERROR!"); + e.printStackTrace(); + + return false; + } + + } + + private String getAlarmTag(String monitorStrategyName, String alarmLevel) { + String alarmTag = null; + if (monitorStrategyName.equals("INode Quota") || + monitorStrategyName.equals("Space Quota")) { + alarmTag = alarmLevel; + } else { + if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + alarmLevel, "WARN") == 0) { + alarmTag = "一线处理"; + } else if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + alarmLevel, "CRITICAL") >= 0) { + alarmTag = "二线处理"; + } else { + alarmTag = "二线处理"; + } + } + return alarmTag; + } + + private String[] getReceivers(String infoLevel, Configuration conf) { + /* + 优先选择监控策略报警接受者，如无，则选择默认配置 + */ + String[] receivers = getWorkerArgs(); + if (receivers != null) { + System.out.println("Receiver ：" + receivers); + return receivers; + } + + System.out.println("### Used default receiver"); + + if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + infoLevel, "WARN") == 0) { + receivers = conf.getStrings( + "sre.mobile.reporter.warning.level.alarm.receivers"); + } else if (MonitorStrategyAttachedTaskAdapter.compareAlarmLevel( + infoLevel, "CRITICAL") >= 0) { + receivers = conf.getStrings( + "sre.mobile.reporter.critical.level.alarm.receivers"); + } else { + receivers = getWorkerArgs(); + } + return receivers; + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTask.java b/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTask.java new file mode 100644 index 0000000..3d83369 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTask.java @@ -0,0 +1,77 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + + +/** + * @author guoyezhi + * + */ +public abstract class MonitorStrategyAttachedTask implements Runnable { + + private String taskWorker = null; + + private String[] workerArgs = null; + + private MonitorStrategyAttachedTaskArguments args = null; + + + /** + * @param taskWorker + * @param workerArgs + * @param args + */ + public MonitorStrategyAttachedTask( + String taskWorker, + String[] workerArgs, + MonitorStrategyAttachedTaskArguments args) { + this.taskWorker = taskWorker; + this.workerArgs = workerArgs; + this.args = args; + } + + + /** + * @param taskWorker the taskWorker to set + */ + public void setTaskWorker(String taskWorker) { + this.taskWorker = taskWorker; + } + + /** + * @return the taskWorker + */ + public String getTaskWorker() { + return taskWorker; + } + + /** + * @param workerArgs the workerArgs to set + */ + public void setWorkerArgs(String[] workerArgs) { + this.workerArgs = workerArgs; + } + + /** + * @return the workerArgs + */ + public String[] getWorkerArgs() { + return workerArgs; + } + + /** + * @param args the args to set + */ + public void setAttachedTaskArguments(MonitorStrategyAttachedTaskArguments args) { + this.args = args; + } + + /** + * @return the args + */ + public MonitorStrategyAttachedTaskArguments getAttachedTaskArguments() { + return args; + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTaskAdapter.java b/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTaskAdapter.java new file mode 100644 index 0000000..bda72f6 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTaskAdapter.java @@ -0,0 +1,271 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import java.util.Date; + +/** + * @author guoyezhi + * + */ +public class MonitorStrategyAttachedTaskAdapter { + + private final Class taskType; + + private int continuousOccurrence = -1; + + private final int continuousLimit; + + /* + * added by Jiangtao02 + */ + private final long intervalLimit; + private long intervalOld = -1; + + private final String taskWorker; + + private String[] workerArgs; + + private MonitorStrategyAttachedTaskArguments lastTaskArgs = null; + + private MonitorStrategyAttachedTaskArguments lastTriggeredTaskArgs = null; + + + /** + * @param continuousLimit + */ + public MonitorStrategyAttachedTaskAdapter( + Class taskType, + int continuousLimit, + String taskWorker, + String workerArgsConf, + long intervalLimit) { + this.taskType = taskType; + this.continuousOccurrence = 0; + this.continuousLimit = continuousLimit; + this.taskWorker = taskWorker; + if (workerArgsConf == null) { + this.workerArgs = null; + } else { + this.workerArgs = workerArgsConf.split(","); + } + this.intervalLimit = intervalLimit; + } + + + /** + * @return the taskWorker + */ + public String getTaskWorker() { + return taskWorker; + } + + /** + * @param continuousOccurrence the continuousOccurrence to set + */ + public synchronized void setContinuousOccurrence(int continuousOccurrence) { + this.continuousOccurrence = continuousOccurrence; + } + + /** + * @return the continuousOccurrence + */ + public synchronized int getContinuousOccurrence() { + return continuousOccurrence; + } + + public synchronized void clearContinuousOccurrence() { + + if (continuousOccurrence > 0 + && shouldReportRecovery(lastTriggeredTaskArgs)) { + + lastTriggeredTaskArgs.setKeyInfo( + "恢复正常（异常持续" + continuousOccurrence + "分钟）"); + lastTriggeredTaskArgs.setFullInfo( + "恢复正常（异常持续" + continuousOccurrence + "分钟）"); + lastTriggeredTaskArgs.setTimestamp(new Date()); + + MonitorStrategyAttachedTask task = null; + try { + task = this.taskType.getConstructor( + String.class, String[].class, + MonitorStrategyAttachedTaskArguments.class).newInstance( + taskWorker, workerArgs, lastTriggeredTaskArgs); + new Thread(task).start(); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + } + + continuousOccurrence = 0; + lastTriggeredTaskArgs = null; + + } + + private boolean shouldReportRecovery(MonitorStrategyAttachedTaskArguments args) { + + /* + * In some cases, not getting worse do not mean recovery, + * e.g. HDFS_DatanodeAliveToDead_MON & MapRed_TasktrackerAliveToDead_MON + */ + if (args.getMonitorStrategyName().contains("AliveToDead")) { + return false; + } + + return true; + + } + + public synchronized void runTask(MonitorStrategyAttachedTaskArguments taskArgs) { + + boolean shouldTrigger = false; + + if (continuousLimit <= 0) { + continuousOccurrence++; + shouldTrigger = true; + } else if (continuousLimit > 0) { + if (lastTaskArgs == null) { + continuousOccurrence++; + } else if (taskArgs.getTimestamp().after( + lastTaskArgs.getTimestamp())) { + continuousOccurrence++; + } + if ((continuousOccurrence > 0) + && (continuousOccurrence <= continuousLimit)) { + shouldTrigger = true; + intervalOld = System.currentTimeMillis(); + } else if(continuousOccurrence > 0){ + //added by jiangtao02 + if(intervalOld > 0 && intervalLimit > 0){ + long now = System.currentTimeMillis(); + if((now - intervalOld) > intervalLimit * 1000) { + /* + * Debug + */ + System.out.println("##Debug: " + (now - intervalOld) + " < Limit:" + intervalLimit * 1000); + shouldTrigger = true; + intervalOld = now; + } + } + } + } + + + /* + * In case the situation of monitor strategy is getting worse + * after sub-tasks have been triggered more than limit times, + * we add the following codes to support high priority override. + */ + if (!shouldTrigger && (lastTriggeredTaskArgs != null)) { + if (compareAlarmLevel( + taskArgs.getAlarmLevel(), + lastTriggeredTaskArgs.getAlarmLevel()) > 0) { + continuousOccurrence = 1; + shouldTrigger = true; + intervalOld = System.currentTimeMillis() * 1000; + } + } + + lastTaskArgs = taskArgs; + + if (shouldTrigger) { + MonitorStrategyAttachedTask task = null; + try { + task = this.taskType.getConstructor( + String.class, String[].class, + MonitorStrategyAttachedTaskArguments.class).newInstance( + taskWorker, workerArgs, taskArgs); + new Thread(task).start(); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + lastTriggeredTaskArgs = taskArgs; + } + + } + + + /* + * + * + */ + public static final int ALARM_LEVEL_DETECTION_PASSED = 0x0000; + + public static final int ALARM_LEVEL_ACCQ_TIMEOUT = 0x4000; + + public static final int ALARM_LEVEL_UNINIT = 0x8000; + public static final int ALARM_LEVEL_INFO = 0x8001; + public static final int ALARM_LEVEL_NOTICE = 0x8002; + public static final int ALARM_LEVEL_WARN = 0x8003; + public static final int ALARM_LEVEL_CRITICAL = 0x8004; + public static final int ALARM_LEVEL_EMERG = 0x8005; + + public static int compareAlarmLevel(int val1, int val2) { + if (val1 == val2) { + return 0; + } else if (val1 > val2) { + return 1; + } else { + return -1; + } + } + + public static int compareAlarmLevel(String valStr1, String valStr2) { + int val1 = getAlarmLevelValueFromString(valStr1); + int val2 = getAlarmLevelValueFromString(valStr2); + if (val1 == val2) { + return 0; + } else if (val1 > val2) { + return 1; + } else { + return -1; + } + } + + public static String getAlarmLevelStringFromValue(int val) { + if (val == ALARM_LEVEL_INFO) { + return "INFO"; + } + else if (val == ALARM_LEVEL_NOTICE) { + return "NOTICE"; + } + else if (val == ALARM_LEVEL_WARN) { + return "WARN"; + } + else if (val == ALARM_LEVEL_CRITICAL) { + return "CRITICAL"; + } + else if (val == ALARM_LEVEL_EMERG) { + return "EMERG"; + } + else { + return "UNINIT"; + } + } + + public static int getAlarmLevelValueFromString(String str) { + if (str.equals("INFO")) { + return ALARM_LEVEL_INFO; + } + else if (str.equals("NOTICE")) { + return ALARM_LEVEL_NOTICE; + } + else if (str.equals("WARN") || str.equals("WARNING")) { + return ALARM_LEVEL_WARN; + } + else if (str.equals("CRITICAL")) { + return ALARM_LEVEL_CRITICAL; + } + else if (str.equals("EMERG")) { + return ALARM_LEVEL_EMERG; + } + else { + return ALARM_LEVEL_UNINIT; + } + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTaskAdapterGenerater.java b/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTaskAdapterGenerater.java new file mode 100644 index 0000000..4379cc8 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTaskAdapterGenerater.java @@ -0,0 +1,107 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; + +/** + * @author guoyezhi + * + */ +public class MonitorStrategyAttachedTaskAdapterGenerater { + + @SuppressWarnings("unchecked") + public static MonitorStrategyAttachedTaskAdapter + generateMonitorStrategyAttachedTaskAdapter(Element strategy) { + + String typeConf = null; + String limitConf = null; + String workerConf = null; + String argsConf = null; + + //added by jiangtao + String intervalConf = null; + + NodeList nodes = strategy.getChildNodes(); + for (int i = 0; i < nodes.getLength(); i++) { + Node node = nodes.item(i); + if (!(node instanceof Element)) + continue; + Element e = (Element) node; + if ("type".equals(e.getTagName()) && e.hasChildNodes()) + typeConf = ((Text)e.getFirstChild()).getData().trim(); + if ("limit".equals(e.getTagName()) && e.hasChildNodes()) { + if (e.getFirstChild() == null) { + System.err.println("e.getFirstChild() == null"); + } else if (((Text)e.getFirstChild()).getData() == null) { + System.err.println("((Text)e.getFirstChild()).getData() == null"); + } + limitConf = ((Text)e.getFirstChild()).getData().trim(); + } + if ("interval".equals(e.getTagName()) && e.hasChildNodes()) { + if (e.getFirstChild() == null) { + System.err.println("e.getFirstChild() == null"); + } else if (((Text)e.getFirstChild()).getData() == null) { + System.err.println("((Text)e.getFirstChild()).getData() == null"); + } + intervalConf = ((Text)e.getFirstChild()).getData().trim(); + } + if ("worker".equals(e.getTagName()) && e.hasChildNodes()) + workerConf = ((Text)e.getFirstChild()).getData().trim(); + if ("args".equals(e.getTagName()) && e.hasChildNodes()) + argsConf = ((Text)e.getFirstChild()).getData().trim(); + } + + Class taskType = null; + if (typeConf != null) { + try { + taskType = + (Class) Class.forName( + "hma.monitor.strategy.trigger.task.Attached" + + typeConf); + } catch (ClassNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new RuntimeException(); + } + } + + int continuousLimit = -1; + int intervalLimit = -1; + if (limitConf != null) { + continuousLimit = Integer.parseInt(limitConf); + } + if (intervalConf != null) { + intervalLimit = Integer.parseInt(intervalConf); + } + + if (taskType != null && (continuousLimit >= 0) && workerConf != null) { + System.out.println("XXX typeConf = " + typeConf); + System.out.println("XXX limitConf = " + limitConf); + System.out.println("XXX workerConf = " + workerConf); + System.out.println("XXX argsConf = " + argsConf); + return new MonitorStrategyAttachedTaskAdapter( + taskType, continuousLimit, workerConf, argsConf, intervalLimit); + } + + System.out.println("XXX typeConf = " + typeConf); + System.out.println("XXX limitConf = " + limitConf); + System.out.println("XXX workerConf = " + workerConf); + System.out.println("XXX argsConf = " + argsConf); + + return null; + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTaskArguments.java b/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTaskArguments.java new file mode 100644 index 0000000..87a2c39 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/MonitorStrategyAttachedTaskArguments.java @@ -0,0 +1,106 @@ +/** + * + */ +package hma.monitor.strategy.trigger.task; + +import java.util.Date; + +/** + * @author guoyezhi + * add additionalInfo by chiwen01 + * + */ +public class MonitorStrategyAttachedTaskArguments { + + private String targetSystemName = null; + + private String monitorStrategyName = null; + + private String alarmLevel = null; + + private String keyInfo = null; + + private String fullInfo = null; + + private Date timestamp = null; + + private String additionalInfo = null; + + + public MonitorStrategyAttachedTaskArguments( + String targetSystemName, + String monitorStrategyName, + String alarmLevel, + String keyInfo, + String fullInfo, + Date timestamp, + String additionalInfo) { + this.targetSystemName = targetSystemName; + this.monitorStrategyName = monitorStrategyName; + this.alarmLevel = alarmLevel; + this.keyInfo = keyInfo; + this.fullInfo = fullInfo; + this.timestamp = timestamp; + this.additionalInfo = additionalInfo; + } + + public MonitorStrategyAttachedTaskArguments( + String targetSystemName, + String monitorStrategyName, + String alarmLevel, + String keyInfo, + String fullInfo, + Date timestamp) { + this.targetSystemName = targetSystemName; + this.monitorStrategyName = monitorStrategyName; + this.alarmLevel = alarmLevel; + this.keyInfo = keyInfo; + this.fullInfo = fullInfo; + this.timestamp = timestamp; + } + + public String getTargetSystemName() { + return targetSystemName; + } + + public String getMonitorStrategyName() { + return monitorStrategyName; + } + + public String getAlarmLevel() { + return alarmLevel; + } + + public String getKeyInfo() { + return keyInfo; + } + + public void setKeyInfo(String keyInfo) { + this.keyInfo = keyInfo; + } + + public String getFullInfo() { + return fullInfo; + } + + public void setFullInfo(String fullInfo) { + this.fullInfo = fullInfo; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public String getAdditionalInfo() { + return additionalInfo; + } + + public void setAdditionalInfo() { + this.additionalInfo = additionalInfo; + } + +} diff --git a/src/java/hma/monitor/strategy/trigger/task/SendMailTest.java b/src/java/hma/monitor/strategy/trigger/task/SendMailTest.java new file mode 100644 index 0000000..b4aab11 --- /dev/null +++ b/src/java/hma/monitor/strategy/trigger/task/SendMailTest.java @@ -0,0 +1,46 @@ +package hma.monitor.strategy.trigger.task; + +import org.apache.commons.mail.EmailException; +import org.apache.commons.mail.SimpleEmail; + +/** + * Created by weizhonghui on 2016/11/19. + */ +public class SendMailTest { + private void doSendMail(String[] receivers, String subject, String body) { + + try { + //String hostname = "yz750.hadoop.data.sina.com.cn"; + SimpleEmail email = new SimpleEmail(); + for (int i = 0; i < receivers.length; i++){ + email.addTo(receivers[i]); + } + email.setCharset("gbk"); + //String mfrom = System.getProperty("user.name") + "@"+ hostname; + String mfrom = "monitor@hma.org"; + System.out.println("log : " + mfrom); + email.setFrom(mfrom); + email.setAuthentication("boyan5@staff.sina.com.cn", "ddzs%%017"); + + email.setHostName("mail.staff.sina.com.cn"); + email.setSubject(subject); + email.setMsg(body); + email.send(); + } catch (EmailException ee) { + String tmp = ""; + for (int i = 0; i < receivers.length; i++){ + tmp = tmp + receivers[i]; + } + System.err.println("EmailException: mail is = " + tmp); + throw new RuntimeException(ee); + } + + } + public static void main(String [] args ){ + String [] receivers = {"boyan5@staff.sina.com.cn"}; + String subject = "Hma EmailSend Experiment"; + String body = "Last week I went to the threatre. I had a very good seat and the play was very interesting."; + new SendMailTest().doSendMail(receivers,subject,body); + } +} + diff --git a/src/java/hma/util/BuiltinPrototypesInjector.java b/src/java/hma/util/BuiltinPrototypesInjector.java new file mode 100644 index 0000000..e64cc26 --- /dev/null +++ b/src/java/hma/util/BuiltinPrototypesInjector.java @@ -0,0 +1,49 @@ +package hma.util; + +import hma.monitor.MonitorManager; +import hma.monitor.collection.BuiltinMonitorItemPrototype; + +import java.util.Map; + +public class BuiltinPrototypesInjector { + public static void BuiltinPrototypesInjector(Map NowBuiltinPrototypes) { + + String DstName = "MapRed_RPC_ComputingStatus_AvailabilityProbingJob"; + String PhyQueueName = MonitorManager.getGlobalConf().get("mapred.abaci.phyqueuename", "null"); + + if (PhyQueueName.equals("null") || PhyQueueName.equals("")) { + return; + } + + if(PhyQueueName != null ){ + String QueueNames[] = PhyQueueName.split(";"); + + BuiltinMonitorItemPrototype prototype = NowBuiltinPrototypes.get(DstName); + + for (String QueueName : QueueNames) { + + BuiltinMonitorItemPrototype prototypetmp = + new BuiltinMonitorItemPrototype(DstName + ":" + QueueName, + prototype.getType(), prototype.getPeriod(), prototype.getDescription(), + prototype.getMonitorItemWorkerClass().getName(), + prototype.getRawDataTypeName() + "." + QueueName); + + NowBuiltinPrototypes.put(DstName + ":" + QueueName, prototypetmp); + } + } + + + + + //Iterator> iter = NowBuiltinPrototypes.entrySet().iterator(); + // while (iter.hasNext()) { + // Map.Entry entry = iter.next(); + // String name = entry.getKey(); + //System.out.println("kain's buildin item : " + name); + // } + return; + + + } +} + diff --git a/src/java/hma/util/CharsetConverter.java b/src/java/hma/util/CharsetConverter.java new file mode 100644 index 0000000..657a9cd --- /dev/null +++ b/src/java/hma/util/CharsetConverter.java @@ -0,0 +1,119 @@ +/** + * + */ +package hma.util; + +import java.io.UnsupportedEncodingException; + +/** + * @author guoyezhi + * + */ +public class CharsetConverter { + + /** 7位ASCII字符，也叫作ISO646-US、Unicode字符集的基本拉丁块 */ + public static final String US_ASCII = "US-ASCII"; + /** ISO拉丁字母表 No.1，也叫做ISO-LATIN-1 */ + public static final String ISO_8859_1 = "ISO-8859-1"; + /** 8 位 UCS 转换格式 */ + public static final String UTF_8 = "UTF-8"; + /** 16 位 UCS 转换格式，Big Endian(最低地址存放高位字节）字节顺序 */ + public static final String UTF_16BE = "UTF-16BE"; + /** 16 位 UCS 转换格式，Litter Endian（最高地址存放地位字节）字节顺序 */ + public static final String UTF_16LE = "UTF-16LE"; + /** 16 位 UCS 转换格式，字节顺序由可选的字节顺序标记来标识 */ + public static final String UTF_16 = "UTF-16"; + /** 中文超大字符集 **/ + public static final String GBK = "GBK"; + + public static final String GB2312 = "GB2312"; + + /** 将字符编码转换成US-ASCII码 */ + public String toASCII(String str) throws UnsupportedEncodingException { + return changeCharset(str, US_ASCII); + } + + /** 将字符编码转换成ISO-8859-1 */ + public String toISO_8859_1(String str) throws UnsupportedEncodingException { + return changeCharset(str, ISO_8859_1); + } + + /** 将字符编码转换成UTF-8 */ + public String toUTF_8(String str) throws UnsupportedEncodingException { + return changeCharset(str, UTF_8); + } + + /** 将字符编码转换成UTF-16BE */ + public String toUTF_16BE(String str) throws UnsupportedEncodingException { + return changeCharset(str, UTF_16BE); + } + + /** 将字符编码转换成UTF-16LE */ + public String toUTF_16LE(String str) throws UnsupportedEncodingException { + return changeCharset(str, UTF_16LE); + } + + /** 将字符编码转换成UTF-16 */ + public String toUTF_16(String str) throws UnsupportedEncodingException { + return changeCharset(str, UTF_16); + } + + /** 将字符编码转换成GBK */ + public String toGBK(String str) throws UnsupportedEncodingException { + return changeCharset(str, GBK); + } + + /** 将字符编码转换成GB2312 */ + public String toGB2312(String str) throws UnsupportedEncodingException { + return changeCharset(str, GB2312); + } + + /** + * 字符串编码转换的实现方法 + * + * @param str + * 待转换的字符串 + * @param newCharset + * 目标编码 + */ + public static String changeCharset(String str, String newCharset) + throws UnsupportedEncodingException { + if (str != null) { + // 用默认字符编码解码字符串。与系统相关，中文windows默认为GB2312 + byte[] bs = str.getBytes(); + return new String(bs, newCharset); // 用新的字符编码生成字符串 + } + return null; + } + + /** + * 字符串编码转换的实现方法 + * + * @param str + * 待转换的字符串 + * @param oldCharset + * 源字符集 + * @param newCharset + * 目标字符集 + */ + public static String changeCharset(String str, String oldCharset, String newCharset) + throws UnsupportedEncodingException { + if (str != null) { + // 用源字符编码解码字符串 + byte[] bs = str.getBytes(oldCharset); + return new String(bs, newCharset); + } + return null; + } + + + /** + * @param args + * @throws UnsupportedEncodingException + */ + public static void main(String[] args) throws UnsupportedEncodingException { + String str = "采集超时"; + System.out.println(new String(str.getBytes("gbk"), "utf-8")); + } + +} diff --git a/src/java/hma/util/DaStrategyInjector.java b/src/java/hma/util/DaStrategyInjector.java new file mode 100644 index 0000000..2495881 --- /dev/null +++ b/src/java/hma/util/DaStrategyInjector.java @@ -0,0 +1,38 @@ +package hma.util; + +import hma.monitor.MonitorManager; +import hma.monitor.strategy.MonitorStrategyPrototype; +import hma.monitor.strategy.MonitorStrategyPrototype.DataAcquisitionStrategy; + +import java.util.Map; + +public class DaStrategyInjector { + public static void DaStrategyInjector(Map nowStrategyPrototypes){ + String dstStrategyName = "MapRed_ComputingFunctionAvailability_MON"; + String dstDaStrategyMonitorName = "MapRed_RPC_ComputingStatus_AvailabilityProbingJob"; + + String PhyQueueName = MonitorManager.getGlobalConf().get("mapred.abaci.phyqueuename", "null"); + + if (PhyQueueName.equals("null")){ + return; + } + String QueueNames [] = PhyQueueName.split(";"); + + if(dstStrategyName!=null){ + DataAcquisitionStrategy dstDaStrategy = nowStrategyPrototypes.get(dstStrategyName).getDataAcquisitionStrategy(); + for (String QueueName : QueueNames) { + dstDaStrategy.putMeta(dstDaStrategyMonitorName + ":" + QueueName, + dstDaStrategy.getTimeoutMetaByMonitorItemName(dstDaStrategyMonitorName), + dstDaStrategy.getAttachedTaskConfsByMonitorItemName(dstDaStrategyMonitorName)); + + } + } + + + + +// for ( String daName:nowStrategyPrototypes.get(dstStrategyName).getDataAcquisitionStrategy().getAllInvolvedItemNames() ){ +// System.out.println("kain's log from DaStrategyInjector daStrategyName is : " + daName); +// } + } +} diff --git a/src/java/hma/util/EasyWebGet.java b/src/java/hma/util/EasyWebGet.java new file mode 100644 index 0000000..a9da93c --- /dev/null +++ b/src/java/hma/util/EasyWebGet.java @@ -0,0 +1,91 @@ +package hma.util; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import java.net.URL; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.util.EntityUtils; + +public class EasyWebGet { + public static String get (String url) throws ClientProtocolException, IOException { + + //String url = "http://hk-dlb-hdfs.dmop.baidu.com:8070/corrupt_replicas_xml.jsp" ; + + + String rspContent = null ; + HttpClient httpclient = new DefaultHttpClient(); + + HttpGet httpget = new HttpGet(url); + + HttpResponse response = httpclient.execute(httpget); + HttpEntity rspEntity = response.getEntity(); + + if (rspEntity != null) { + rspContent = EntityUtils.toString(rspEntity); + } + + return rspContent; + } + + /** + * add by yangsen01 + * for Hermes. important alerm should call + * + * 发送HttpPost请求 + * @param strURL 服务地址 + * @param params json字符串,例如: "{ \"id\":\"12345\" }" ;其中属性名必须带双引号
+ * @return 成功:返回json字符串
+ */ + public static String post(String strURL, String params) { + System.out.println(strURL); + System.out.println(params); + try { + URL url = new URL(strURL);// 创建连接 + HttpURLConnection connection = (HttpURLConnection) url + .openConnection(); + connection.setDoOutput(true); + connection.setDoInput(true); + connection.setUseCaches(false); + connection.setInstanceFollowRedirects(true); + connection.setRequestMethod("POST"); // 设置请求方式 + + connection.connect(); + OutputStreamWriter out = new OutputStreamWriter( + connection.getOutputStream(), "UTF-8"); // utf-8编码 + out.append(params); + out.flush(); + out.close(); + // 读取响应 + int length = (int) connection.getContentLength();// 获取长度 + InputStream is = connection.getInputStream(); + if (length != -1) { + byte[] data = new byte[length]; + byte[] temp = new byte[512]; + int readLen = 0; + int destPos = 0; + while ((readLen = is.read(temp)) > 0) { + System.arraycopy(temp, 0, data, destPos, readLen); + destPos += readLen; + } + String result = new String(data, "UTF-8"); // utf-8编码 + System.out.println(result); + return result; + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return "error"; // 自定义错误信息 + } + +} diff --git a/src/java/hma/util/HMALogger.java b/src/java/hma/util/HMALogger.java new file mode 100644 index 0000000..4684ecb --- /dev/null +++ b/src/java/hma/util/HMALogger.java @@ -0,0 +1,44 @@ +/** + * + */ +package hma.util; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; + +/** + * @author guoyezhi + * + */ +public class HMALogger { + + private File logFile = null; + + public HMALogger(String logName) { + this.logFile = new File(logName); + } + + public void log(String content) { + PrintWriter writer = null; + try { + writer = new PrintWriter( + new FileWriter(logFile, true)); + writer.println(content); + } catch (IOException ioe) { + ioe.printStackTrace(); + } finally { + writer.close(); + } + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/util/HadoopEmptyFileGenerator.java b/src/java/hma/util/HadoopEmptyFileGenerator.java new file mode 100644 index 0000000..ef2f2ea --- /dev/null +++ b/src/java/hma/util/HadoopEmptyFileGenerator.java @@ -0,0 +1,76 @@ +/** + * + */ +package hma.util; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +/** + * @author guoyezhi + * + */ +public class HadoopEmptyFileGenerator extends Configured { + + int highDirBeginID = 0; + + int midDirBeginID = 0; + + int lowFileBeginID = 0; + + long fileMaxNum = 0; + + public HadoopEmptyFileGenerator() { + super(new Configuration()); + } + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + + HadoopEmptyFileGenerator gen = + new HadoopEmptyFileGenerator(); + + // File Path: /high_dir/mid_dir/low_file + if (args.length >= 4) { + gen.highDirBeginID = Integer.parseInt(args[0]); + gen.midDirBeginID = Integer.parseInt(args[1]); + gen.lowFileBeginID = Integer.parseInt(args[2]); + gen.fileMaxNum = Long.parseLong(args[3]); + } else { + gen.highDirBeginID = 0; + gen.midDirBeginID = 0; + gen.lowFileBeginID = 0; + gen.fileMaxNum = 120000000; + } + + FileSystem srcFs = FileSystem.get(gen.getConf()); + + long cnt = 0; + for (int i = gen.highDirBeginID; i < 9999; i++) { + for (int j = gen.midDirBeginID; j < 1000; j++) { + for (int k = gen.lowFileBeginID; k < 1000; k++) { + + if (++cnt > gen.fileMaxNum) + break; + + String src = "/" + i + + "/" + j + + "/" + k; + Path f = new Path(src); + FSDataOutputStream out = srcFs.create(f); + out.close(); + } + } + } + + } + +} diff --git a/src/java/hma/util/HmaStatusJsonIf.java b/src/java/hma/util/HmaStatusJsonIf.java new file mode 100644 index 0000000..601de8e --- /dev/null +++ b/src/java/hma/util/HmaStatusJsonIf.java @@ -0,0 +1,98 @@ +package hma.util; +import hma.web.HMAJspHelper; + +import java.io.IOException; +import java.io.PrintWriter; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; +import org.json.JSONObject; + +public class HmaStatusJsonIf extends HttpServlet { + + private static final long serialVersionUID = 1L; + + public void doPost(HttpServletRequest req, HttpServletResponse res) throws IOException{ + + res.setContentType("text/html"); + PrintWriter out = res.getWriter(); + out.println("Got Post"); + } + + public void doGet(HttpServletRequest request, HttpServletResponse res) throws IOException{ + res.setContentType("text/html"); + PrintWriter out = res.getWriter(); + HMAJspHelper helper = new HMAJspHelper(); + SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd" + " " + "HH:mm:ss"); + + String clusterName = request.getParameter("clusterName"); + String strategyName = request.getParameter("strategyName"); + if (clusterName == null || clusterName.equals("")) throw new NullPointerException("No clustername!"); + if (strategyName == null || strategyName.equals("")) throw new NullPointerException("No strategyName!"); + + // clusterName = clusterName.toUpperCase(); + + String endTime = dateFormat.format(new Date()); + String begTime = dateFormat.format(new Date( (new Date()).getTime() - 3600 * 1000)); + //out.println("cluster is: " + clusterName + " time is " + " to" + dateFormat.format(new Date()) ); + //out.println("cluster is: " + clusterName + " time is " + begTime + " to " + endTime.toString() ); + + Map checkResult = null; + + try { + checkResult = helper.checkDetectionResults(clusterName, begTime, endTime); + } catch (SQLException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + int detectionResult = checkResult.get(strategyName); + String detectionResultStr = null; + + if (detectionResult == HMAJspHelper.DetectionResult_NODATA) { + detectionResultStr = + "NODATA"; + } else if (detectionResult == HMAJspHelper.DetectionResult_NORMAL) { + detectionResultStr = + "NORMAL"; + } else { + detectionResultStr = "ANOMALY"; + } + + //out.println("cluster is: " + clusterName + " strategyName is: " + strategyName + " result is: " + detectionResultStr ); + //out.print( detectionResultStr ); + //JSONArray statusArr = new JSONArray(); + JSONObject clusterObj = new JSONObject(); + + JSONObject strategyObj = new JSONObject(); + try { + strategyObj.put(strategyName, detectionResultStr); + } catch (JSONException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + //statusArr.put(strategyObj); + //clusterObj.put(clusterName,statusArr ); + + try { + clusterObj.put(clusterName,strategyObj); + } catch (JSONException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + + //out.print(clusterObj.toString()); + //out.print("{\"" + clusterName + "\":{\"" + strategyName + "\":\"" + detectionResultStr + "\"}}"); + out.print(clusterObj.toString()); + } + +} diff --git a/src/java/hma/util/HttpClientHelper.java b/src/java/hma/util/HttpClientHelper.java new file mode 100644 index 0000000..c5dad61 --- /dev/null +++ b/src/java/hma/util/HttpClientHelper.java @@ -0,0 +1,200 @@ +/** + * + */ +package hma.util; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpClient; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIUtils; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * @author guoyezhi + */ +public class HttpClientHelper { + public static final int ERROR_STATUS_CODE = -1; + public static final int OK_STATUS_CODE = 200; + + /** + * @param scheme + * @param host + * @param port + * @param path + * @param params + * @param entityEncoding + * @return + * @throws java.net.URISyntaxException + * @throws org.apache.http.client.ClientProtocolException + * + * @throws java.io.IOException + */ + public static String getRequest( + String scheme, + String host, + int port, + String path, + Map params, + String entityEncoding) + throws URISyntaxException, ClientProtocolException, IOException { + + List paramList = new ArrayList(); + Iterator> iter = params.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String param = entry.getKey(); + String value = entry.getValue(); + paramList.add(new BasicNameValuePair(param, value)); + } + URI uri = URIUtils.createURI( + scheme, host, port, path, + URLEncodedUtils.format(paramList, "UTF-8"), null); + HttpGet httpget = new HttpGet(uri); + + HttpClient httpclient = new DefaultHttpClient(); + HttpResponse response = httpclient.execute(httpget); + + String rspContent = null; + HttpEntity rspEntity = response.getEntity(); + if (rspEntity != null) { + rspContent = EntityUtils.toString(rspEntity); + } + EntityUtils.consume(rspEntity); + + return rspContent; + } + + + /** + * @param url + * @return + */ + public static String postRequest(String url) { + throw new UnsupportedOperationException(); + } + + /** + * @param url + * @param params + * @param entityEncoding + * @return + * @throws java.io.UnsupportedEncodingException + * + * @throws org.apache.http.client.ClientProtocolException + * + * @throws java.io.IOException + */ + public static String postRequest( + String url, + Map params, + String entityEncoding) + throws UnsupportedEncodingException, ClientProtocolException, IOException { + + String rspContent = null; + + HttpClient httpclient = new DefaultHttpClient(); + HttpPost httppost = new HttpPost(url); + List paramList = new ArrayList(); + Iterator> iter = params.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String param = entry.getKey(); + String value = entry.getValue(); + paramList.add(new BasicNameValuePair(param, value)); + } + UrlEncodedFormEntity reqEntity = + new UrlEncodedFormEntity(paramList, entityEncoding); + httppost.setEntity(reqEntity); + HttpResponse response = httpclient.execute(httppost); + HttpEntity rspEntity = response.getEntity(); + if (rspEntity != null) { + rspContent = EntityUtils.toString(rspEntity); + } + EntityUtils.consume(rspEntity); + + return rspContent; + } + + /** + * @param url "http://caoliushequ.com?page=1" + * @return int + */ + public static int getStatusCode(String url) { + URI uri = null; + try { + uri = new URI(url); + } catch (URISyntaxException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + return ERROR_STATUS_CODE; + } + HttpGet httpget = new HttpGet(uri); + HttpClient httpclient = new DefaultHttpClient(); + HttpResponse response = null; + try { + response = httpclient.execute(httpget); + } catch (IOException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + LOG.warn("Failed to get http result from " + url); + return ERROR_STATUS_CODE; + } + return response.getStatusLine().getStatusCode(); + + } + + /** + * @param url url to get + * @return String + */ + public static String get(String url) { + URI uri = null; + String rspContent = null; + HttpResponse response = null; + + try { + uri = new URI(url); + } catch (URISyntaxException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + return null; + } + HttpGet httpget = new HttpGet(uri); + HttpClient httpclient = new DefaultHttpClient(); + try { + response = httpclient.execute(httpget); + } catch (IOException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + LOG.warn("Failed to get http result from " + url); + return null; + } + HttpEntity rspEntity = response.getEntity(); + if (rspEntity != null) { + try { + rspContent = EntityUtils.toString(rspEntity); + EntityUtils.consume(rspEntity); + } catch (IOException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + LOG.warn("Failed to get http result from " + url); + return null; + } + } + return rspContent; + } + + +} diff --git a/src/java/hma/util/InspectionDataInjector.java b/src/java/hma/util/InspectionDataInjector.java new file mode 100644 index 0000000..9aeb9b4 --- /dev/null +++ b/src/java/hma/util/InspectionDataInjector.java @@ -0,0 +1,91 @@ +package hma.util; + +import hma.conf.Configuration; +import hma.monitor.MonitorManager; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class InspectionDataInjector { + + + private String dbServer = null; + private String dbName = null; + private String inspectionTable = null; + + private String userName = null; + private String password = null; + private Connection dbConn = null; + + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd" + " " + "HH:mm:ss"); + + private void initConnection() throws SQLException { + + Configuration hmaConf = + MonitorManager.getMonitorManager().getConf(); + + dbServer = hmaConf.get( + "monitor.data.db.server", "yf-2950-016.yf01.baidu.com"); + dbName = hmaConf.get( + "monitor.data.db.name", "hma"); + + inspectionTable = hmaConf.get( + "monitor.inspection.data.db.table", "monitor_inspection"); + userName = hmaConf.get( + "monitor.db.user.name", "root"); + password = hmaConf.get( + "monitor.db.user.pass", "hmainit"); + + dbConn = DriverManager.getConnection( + "jdbc:mysql://" + dbServer + "/" + dbName, + userName, + password); + + /* + dbConn = DriverManager.getConnection( + "jdbc:mysql://" + dbServer + "/" + dbName + + "?useUnicode=true&characterEncoding=utf8", + userName, + password); + */ + } + + public InspectionDataInjector (String clusterName, + String strategyName, + String detectionTimestamp, + String infoLevel, + String infoContent) throws SQLException, ParseException { + this.initConnection(); + + PreparedStatement pstat = null; + String oldTime = dateFormat.format(new Date(dateFormat.parse(detectionTimestamp).getTime() - 3600*1000) ); + + //System.out.println("kain's inspection's log in Injector : " + clusterName + " " + strategyName + " " + detectionTimestamp + " " + infoLevel + " " + infoContent + oldTime); + + pstat = dbConn.prepareStatement("DELETE FROM " + inspectionTable + " WHERE detection_timestamp<" + "'" + oldTime + "'"); + //System.out.println(pstat.toString()); + pstat.executeUpdate(); + + pstat = dbConn.prepareStatement( + "INSERT INTO " + inspectionTable + " VALUES ( ?, ?, ?, ?, ?)"); + pstat.setString(1, clusterName); + pstat.setString(2, strategyName); + pstat.setString(3, detectionTimestamp); + pstat.setString(4, infoLevel); + pstat.setString(5, infoContent); + //System.out.println(pstat.toString()); + pstat.executeUpdate(); + + pstat.close(); + + + } + +} diff --git a/src/java/hma/util/JavaExceptionUtil.java b/src/java/hma/util/JavaExceptionUtil.java new file mode 100644 index 0000000..d301e8c --- /dev/null +++ b/src/java/hma/util/JavaExceptionUtil.java @@ -0,0 +1,40 @@ +/** + * + */ +package hma.util; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + * @author guoyezhi + * + */ +public class JavaExceptionUtil { + + /** + * @param e + * @return + * @throws IOException + */ + public static String getExceptionStackTraceMessage(Exception e) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + pw.flush(); + sw.flush(); + return sw.toString(); + } + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + System.out.println( + JavaExceptionUtil.getExceptionStackTraceMessage( + new RuntimeException("hehe"))); + } + +} diff --git a/src/java/hma/util/LOG.java b/src/java/hma/util/LOG.java new file mode 100644 index 0000000..34bac20 --- /dev/null +++ b/src/java/hma/util/LOG.java @@ -0,0 +1,25 @@ +package hma.util; + +public class LOG { + + public static void fatal(String str) { + System.err.println(str); + } + + public static void warn(String str) { + System.out.println(str); + } + + public static void warn(String str, Exception e) { + System.out.println(str + e.getMessage()); + e.printStackTrace(); + } + + public static void info(String str) { + System.out.println(str); + } + + public static void debug(String str) { + System.out.println(str); + } +} diff --git a/src/java/hma/util/Pair.java b/src/java/hma/util/Pair.java new file mode 100644 index 0000000..a342554 --- /dev/null +++ b/src/java/hma/util/Pair.java @@ -0,0 +1,75 @@ +package hma.util; + +/** + * Created with IntelliJ IDEA. + * User: synckey + * Date: 13-9-10 + * Time: 上午11:26 + * To change this template use File | Settings | File Templates. + */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +public class Pair { + private final T1 left; + private final T2 right; + + public Pair(T1 left, T2 right) { + this.left = left; + this.right = right; + } + + public T1 getLeft() { + return left; + } + + public T2 getRight() { + return right; + } + + @Override + public final int hashCode() { + int hashCode = 31 + (left == null ? 0 : left.hashCode()); + return 31 * hashCode + (right == null ? 0 : right.hashCode()); + } + + @Override + public String toString() { + return "(" + left + "," + right + ")"; + } + + public static Pair create(X x, Y y) { + return new Pair(x, y); + } + + public static void main(String args[]) { + Pair p1 = new Pair(1, "test"); + Pair p2 = Pair.create(1, 2.2); + System.out.println(p1.getLeft()); + System.out.println(p1.getRight()); + System.out.println(p2.getLeft()); + System.out.println(p2.getRight()); + System.out.println(p1.toString()); + System.out.println(p2.toString()); + System.out.println(p1.hashCode()); + System.out.println(p2.hashCode()); + + } +} diff --git a/src/java/hma/util/StringHelper.java b/src/java/hma/util/StringHelper.java new file mode 100644 index 0000000..db726c6 --- /dev/null +++ b/src/java/hma/util/StringHelper.java @@ -0,0 +1,23 @@ +/** + * + */ +package hma.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author guoyezhi + * + */ +public class StringHelper { + + public static String removeDuplicateWhitespace(String inputStr) { + String patternStr = "\\s+"; + String replaceStr = " "; + Pattern pattern = Pattern.compile(patternStr); + Matcher matcher = pattern.matcher(inputStr); + return matcher.replaceAll(replaceStr); + } + +} diff --git a/src/java/hma/util/TimedTask.java b/src/java/hma/util/TimedTask.java new file mode 100644 index 0000000..13e9906 --- /dev/null +++ b/src/java/hma/util/TimedTask.java @@ -0,0 +1,149 @@ +/** + * + */ +package hma.util; + +/** + * @author guoyezhi + * + */ +public abstract class TimedTask implements Runnable { + + /** + * This object is used to control access to the TimedTask internals. + */ + final Object lock = new Object(); + + /** + * The state of this task, chosen from the constants below. + */ + int state = VIRGIN; + + /** + * This task has not yet been scheduled. + */ + static final int VIRGIN = 0; + + /** + * This task is scheduled for execution. If it is a non-repeating task, it + * has not yet been executed. + */ + static final int SCHEDULED = 1; + + /** + * This non-repeating task has already executed (or is currently executing) + * and has not been cancelled. + */ + static final int EXECUTED = 2; + + /** + * This task has been cancelled (with a call to TimedTask.cancel). + */ + static final int CANCELLED = 3; + + /** + * Next execution time for this task in the format returned by + * System.currentTimeMillis, assuming this task is scheduled for execution. + * For repeating tasks, this field is updated prior to each task execution. + */ + long nextExecutionTime; + + /** + * Period in milliseconds for repeating tasks. A positive value indicates + * fixed-rate execution. A negative value indicates fixed-delay execution. A + * value of 0 indicates a non-repeating task. + */ + long period = 0; + + + /** + * Creates a new timer task. + */ + protected TimedTask() { + } + + /** + * The action to be performed by this timer task. + */ + public abstract void run(); + + + /** + * Cancels this timed task. If the task has been scheduled for one-time + * execution and has not yet run, or has not yet been scheduled, it will + * never run. If the task has been scheduled for repeated execution, it will + * never run again. (If the task is running when this call occurs, the task + * will run to completion, but will never run again.) + * + *

+ * Note that calling this method from within the run method of a + * repeating timed task absolutely guarantees that the timed task will not + * run again. + * + *

+ * This method may be called repeatedly; the second and subsequent calls + * have no effect. + * + * @return true if this task is scheduled for one-time execution and has not + * yet run, or this task is scheduled for repeated execution. + * Returns false if the task was scheduled for one-time execution + * and has already run, or if the task was never scheduled, or if + * the task was already cancelled. (Loosely speaking, this method + * returns true if it prevents one or more scheduled + * executions from taking place.) + */ + public boolean cancel() { + synchronized (lock) { + boolean result = (state == SCHEDULED); + state = CANCELLED; + return result; + } + } + + /** + * Returns the scheduled execution time of the most recent + * actual execution of this task. (If this method is invoked while + * task execution is in progress, the return value is the scheduled + * execution time of the ongoing task execution.) + * + *

+ * This method is typically invoked from within a task's run method, to + * determine whether the current execution of the task is sufficiently + * timely to warrant performing the scheduled activity: + * + * 

+	 * public void run() {
+	 * 	if (System.currentTimeMillis() - scheduledExecutionTime() >= MAX_TARDINESS)
+	 * 		return; // Too late; skip this execution.
+	 * 	// Perform the task
+	 * }
+	 *
+ * + * This method is typically not used in conjunction with + * fixed-delay execution repeating tasks, as their scheduled + * execution times are allowed to drift over time, and so are not terribly + * significant. + * + * @return the time at which the most recent execution of this task was + * scheduled to occur, in the format returned by Date.getTime(). The + * return value is undefined if the task has yet to commence its + * first execution. + * @see Date#getTime() + */ + public long scheduledExecutionTime() { + synchronized (lock) { + return (period < 0 ? nextExecutionTime + period : nextExecutionTime + - period); + } + } + + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/util/TimedTaskManager.java b/src/java/hma/util/TimedTaskManager.java new file mode 100644 index 0000000..c0130ba --- /dev/null +++ b/src/java/hma/util/TimedTaskManager.java @@ -0,0 +1,443 @@ +/** + * + */ +package hma.util; + +import java.util.Date; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author guoyezhi + */ +public class TimedTaskManager { + + private ExecutorService exec = null; + + /** + * The timed task queue. This data structure is shared with the scheduler + * thread. This manager produces tasks, via its various schedule calls, and the + * scheduler thread consumes, executing timer tasks as appropriate, and removing + * them from the queue when they're obsolete. + */ + private TimedTaskQueue queue = new TimedTaskQueue(); + + /** + * The timer thread. + */ + private SchedulerThread thread = new SchedulerThread(queue); + + /** + * This object causes the manager's task execution thread to exit gracefully + * when there are no live references to the TimedTaskManager object and no + * tasks in the timed queue. It is used in preference to a finalizer on + * TimedTaskManager as such a finalizer would be susceptible to a subclass's + * finalizer forgetting to call it. + */ + @SuppressWarnings("unused") + private Object threadReaper = new Object() { + protected void finalize() throws Throwable { + synchronized (queue) { + thread.newTasksMayBeScheduled = false; + queue.notify(); // In case queue is empty. + } + } + }; + + + /** + * This ID is used to generate thread names. + */ + private static AtomicInteger nextSerialNumber = new AtomicInteger(0); + + private static synchronized int serialNumber() { + return nextSerialNumber.addAndGet(1); + } + + /** + * Creates a new timed task manager. + * + * @see Thread + * @see #cancel() + */ + public TimedTaskManager() { + this("TimedTaskManager-" + serialNumber(), 128); + } + + /** + * Creates a new timed task manager whose associated thread has the + * specified name. + * + * @param name the name of the associated thread + * @throws NullPointerException if name is null + * @see Thread#getName() + */ + public TimedTaskManager(String name) { + this(name, 128); + } + + /** + * Creates a new timed task manager. + * + * @param nThreads the number of threads in the TimedTaskManager's timed + * task thread pool + * @throws NullPointerException if name is null + * @see Thread#getName() + */ + public TimedTaskManager(int nThreads) { + this("TimedTaskManager-" + serialNumber(), nThreads); + } + + /** + * Creates a new timed task manager whose associated thread has the + * specified name. + * + * @param name the name of the associated thread + * @param nThreads the number of threads in the TimedTaskManager's timed + * task thread pool + * @throws NullPointerException if name is null + * @see Thread#getName() + */ + public TimedTaskManager(String name, int nThreads) { + exec = Executors.newFixedThreadPool(nThreads); + thread.setName(name); + thread.start(); + } + + + /** + * Schedules the specified task for execution after the specified delay. + * + * @param task task to be scheduled. + * @param delay delay in milliseconds before task is to be executed. + * @throws IllegalArgumentException if delay is negative, or + * delay + System.currentTimeMillis() is negative. + * @throws IllegalStateException if task was already scheduled or cancelled, or timer was + * cancelled. + */ + public void schedule(TimedTask task, long delay) { + if (delay < 0) + throw new IllegalArgumentException("Negative delay."); + sched(task, System.currentTimeMillis() + delay, 0); + } + + /** + * Schedules the specified task for execution at the specified time. If the + * time is in the past, the task is scheduled for immediate execution. + * + * @param task task to be scheduled. + * @param time time at which task is to be executed. + * @throws IllegalArgumentException if time.getTime() is negative. + * @throws IllegalStateException if task was already scheduled or cancelled, timer was + * cancelled, or timer thread terminated. + */ + public void schedule(TimedTask task, Date time) { + sched(task, time.getTime(), 0); + } + + /** + * Schedules the specified task for repeated fixed-delay execution, + * beginning after the specified delay. Subsequent executions take place at + * approximately regular intervals separated by the specified period. + *

+ *

+ * In fixed-delay execution, each execution is scheduled relative to the + * actual execution time of the previous execution. If an execution is + * delayed for any reason (such as garbage collection or other background + * activity), subsequent executions will be delayed as well. In the long + * run, the frequency of execution will generally be slightly lower than the + * reciprocal of the specified period (assuming the system clock underlying + * Object.wait(long) is accurate). + * + * @param task task to be scheduled. + * @param delay delay in milliseconds before task is to be executed. + * @param period time in milliseconds between successive task executions. + * @throws IllegalArgumentException if delay is negative, or + * delay + System.currentTimeMillis() is negative. + * @throws IllegalStateException if task was already scheduled or cancelled, timer was + * cancelled, or timer thread terminated. + */ + public void schedule(TimedTask task, long delay, long period) { + if (delay < 0) + throw new IllegalArgumentException("Negative delay."); + if (period <= 0) + throw new IllegalArgumentException("Non-positive period."); + sched(task, System.currentTimeMillis() + delay, -period); + } + + /** + * Schedules the specified task for repeated fixed-delay execution, + * beginning at the specified time. Subsequent executions take place at + * approximately regular intervals, separated by the specified period. + *

+ *

+ * In fixed-delay execution, each execution is scheduled relative to the + * actual execution time of the previous execution. If an execution is + * delayed for any reason (such as garbage collection or other background + * activity), subsequent executions will be delayed as well. In the long + * run, the frequency of execution will generally be slightly lower than the + * reciprocal of the specified period (assuming the system clock underlying + * Object.wait(long) is accurate). + * + * @param task task to be scheduled. + * @param firstTime First time at which task is to be executed. + * @param period time in milliseconds between successive task executions. + * @throws IllegalArgumentException if time.getTime() is negative. + * @throws IllegalStateException if task was already scheduled or cancelled, timer was + * cancelled, or timer thread terminated. + */ + public void schedule(TimedTask task, Date firstTime, long period) { + if (period <= 0) + throw new IllegalArgumentException("Non-positive period."); + sched(task, firstTime.getTime(), -period); + } + + /** + * Schedules the specified task for repeated fixed-rate execution, + * beginning after the specified delay. Subsequent executions take place at + * approximately regular intervals, separated by the specified period. + *

+ *

+ * In fixed-rate execution, each execution is scheduled relative to the + * scheduled execution time of the initial execution. If an execution is + * delayed for any reason (such as garbage collection or other background + * activity), two or more executions will occur in rapid succession to + * "catch up." In the long run, the frequency of execution will be exactly + * the reciprocal of the specified period (assuming the system clock + * underlying Object.wait(long) is accurate). + * + * @param task task to be scheduled. + * @param delay delay in milliseconds before task is to be executed. + * @param period time in milliseconds between successive task executions. + * @throws IllegalArgumentException if delay is negative, or + * delay + System.currentTimeMillis() is negative. + * @throws IllegalStateException if task was already scheduled or cancelled, timer was + * cancelled, or timer thread terminated. + */ + public void scheduleAtFixedRate(TimedTask task, long delay, long period) { + if (delay < 0) + throw new IllegalArgumentException("Negative delay."); + if (period <= 0) + throw new IllegalArgumentException("Non-positive period."); + sched(task, System.currentTimeMillis() + delay, period); + } + + /** + * Schedules the specified task for repeated fixed-rate execution, + * beginning at the specified time. Subsequent executions take place at + * approximately regular intervals, separated by the specified period. + *

+ *

+ * In fixed-rate execution, each execution is scheduled relative to the + * scheduled execution time of the initial execution. If an execution is + * delayed for any reason (such as garbage collection or other background + * activity), two or more executions will occur in rapid succession to + * "catch up." In the long run, the frequency of execution will be exactly + * the reciprocal of the specified period (assuming the system clock + * underlying Object.wait(long) is accurate). + * + * @param task task to be scheduled. + * @param firstTime First time at which task is to be executed. + * @param period time in milliseconds between successive task executions. + * @throws IllegalArgumentException if time.getTime() is negative. + * @throws IllegalStateException if task was already scheduled or cancelled, timer was + * cancelled, or timer thread terminated. + */ + public void scheduleAtFixedRate(TimedTask task, Date firstTime, long period) { + if (period <= 0) + throw new IllegalArgumentException("Non-positive period."); + sched(task, firstTime.getTime(), period); + } + + /** + * Schedule the specified timed task for execution at the specified time + * with the specified period, in milliseconds. If period is positive, the + * task is scheduled for repeated execution; if period is zero, the task is + * scheduled for one-time execution. Time is specified in Date.getTime() + * format. This method checks timer state, task state, and initial execution + * time, but not period. + * + * @throws IllegalArgumentException if time() is negative. + * @throws IllegalStateException if task was already scheduled or cancelled, timer was + * cancelled, or timer thread terminated. + */ + private void sched(TimedTask task, long time, long period) { + if (time < 0) + throw new IllegalArgumentException("Illegal execution time."); + + synchronized (queue) { + if (!thread.newTasksMayBeScheduled) + throw new IllegalStateException("Timer already cancelled."); + + synchronized (task.lock) { + if (task.state != TimedTask.VIRGIN) + throw new IllegalStateException( + "Task already scheduled or cancelled"); + task.nextExecutionTime = time; + task.period = period; + task.state = TimedTask.SCHEDULED; + } + + queue.add(task); + if (queue.getMin() == task) + queue.notify(); + } + } + + /** + * Terminates this manager, discarding any currently scheduled tasks. Does not + * interfere with a currently executing task (if it exists). Once a manager + * has been terminated, its scheduler thread terminates gracefully, and no + * more tasks may be scheduled on it. + *

+ *

+ * This method may be called repeatedly; the second and subsequent calls + * have no effect. + */ + public void terminate() { + synchronized (queue) { + thread.newTasksMayBeScheduled = false; + queue.clear(); + queue.notify(); // In case queue was already empty. + } + } + + /** + * Removes all cancelled tasks from this timer's task queue. Calling this + * method has no effect on the behavior of the timer, but eliminates the + * references to the cancelled tasks from the queue. If there are no + * external references to these tasks, they become eligible for garbage + * collection. + *

+ *

+ * Most programs will have no need to call this method. It is designed for + * use by the rare application that cancels a large number of tasks. Calling + * this method trades time for space: the runtime of the method may be + * proportional to n + c log n, where n is the number of tasks in the queue + * and c is the number of cancelled tasks. + *

+ *

+ * Note that it is permissible to call this method from within a a task + * scheduled on this timer. + */ + public int purge() { + int result = 0; + + synchronized (queue) { + for (int i = queue.size(); i > 0; i--) { + if (queue.get(i).state == TimedTask.CANCELLED) { + queue.quickRemove(i); + result++; + } + } + + if (result != 0) + queue.heapify(); + } + + return result; + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + + + /** + * This "helper class" implements the manager's internal task scheduler + * thread, which waits for tasks on the timer queue, schedules them when + * they fire, reschedules repeating tasks, and removes cancelled tasks + * and spent non-repeating tasks from the queue. + */ + class SchedulerThread extends Thread { + + /** + * This flag is set to false by the reaper to inform us that there + * are no more live references to our TimedTaskManager object. + * Once this flag is true and there are no more tasks in our queue, + * there is no work left for us to do, so we terminate gracefully. + * Note that this field is protected by queue's monitor! + */ + boolean newTasksMayBeScheduled = true; + + /** + * Our TimedTaskManager's queue. We store this reference in preference + * to a reference to the TimedTaskManager so the reference graph remains + * acyclic. Otherwise, the Timer would never be garbage-collected and this + * thread would never go away. + */ + private TimedTaskQueue queue; + + SchedulerThread(TimedTaskQueue queue) { + this.queue = queue; + } + + public void run() { + try { + mainLoop(); + } finally { + // Someone killed this Thread, behave as if Timer cancelled + synchronized (queue) { + newTasksMayBeScheduled = false; + queue.clear(); // Eliminate obsolete references + } + } + } + + /** + * The main timer loop. (See class comment.) + */ + private void mainLoop() { + while (true) { + try { + TimedTask task; + boolean taskFired; + synchronized (queue) { + // Wait for queue to become non-empty + while (queue.isEmpty() && newTasksMayBeScheduled) + queue.wait(); + if (queue.isEmpty()) + break; // Queue is empty and will forever remain; die + + // Queue nonempty; look at first evt and do the right thing + long currentTime, executionTime; + task = queue.getMin(); + synchronized (task.lock) { + if (task.state == TimedTask.CANCELLED) { + queue.removeMin(); + continue; // No action required, poll queue again + } + currentTime = System.currentTimeMillis(); + executionTime = task.nextExecutionTime; + if (taskFired = (executionTime <= currentTime)) { + if (task.period == 0) { // Non-repeating, remove + queue.removeMin(); + task.state = TimedTask.EXECUTED; + } else { // Repeating task, reschedule + queue.rescheduleMin( + task.period < 0 ? (currentTime - task.period) + : (executionTime + task.period)); + } + } + } + if (!taskFired) { // Task hasn't yet fired; wait + queue.wait(executionTime - currentTime); + } + } + if (taskFired) { // Task fired; run it, holding no locks + //task.run(); + exec.execute(task); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + +} diff --git a/src/java/hma/util/TimedTaskQueue.java b/src/java/hma/util/TimedTaskQueue.java new file mode 100644 index 0000000..500b78f --- /dev/null +++ b/src/java/hma/util/TimedTaskQueue.java @@ -0,0 +1,181 @@ +/** + * + */ +package hma.util; + +import java.util.Arrays; + +/** + * This class represents a timed task queue: a priority queue of TimedTasks, + * ordered on nextExecutionTime. Internally this class uses a heap, which offers + * log(n) performance for the add, removeMin and rescheduleMin operations, and + * constant time performance for the getMin operation. + * + * @author guoyezhi + * + */ +class TimedTaskQueue { + + /** + * Priority queue represented as a balanced binary heap: the two children of + * queue[n] are queue[2*n] and queue[2*n+1]. The priority queue is ordered + * on the nextExecutionTime field: The TimedTask with the lowest + * nextExecutionTime is in queue[1] (assuming the queue is nonempty). For + * each node n in the heap, and each descendant of n, d, n.nextExecutionTime + * <= d.nextExecutionTime. + */ + private TimedTask[] queue = new TimedTask[128]; + + /** + * The number of tasks in the priority queue. (The tasks are stored in + * queue[1] up to queue[size]). + */ + private int size = 0; + + /** + * Returns the number of tasks currently on the queue. + */ + int size() { + return size; + } + + /** + * Adds a new task to the priority queue. + */ + void add(TimedTask task) { + // Grow backing store if necessary + if (size + 1 == queue.length) + queue = Arrays.copyOf(queue, 2 * queue.length); + + queue[++size] = task; + fixUp(size); + } + + /** + * Return the "head task" of the priority queue. (The head task is an task + * with the lowest nextExecutionTime.) + */ + TimedTask getMin() { + return queue[1]; + } + + /** + * Return the ith task in the priority queue, where i ranges from 1 (the + * head task, which is returned by getMin) to the number of tasks on the + * queue, inclusive. + */ + TimedTask get(int i) { + return queue[i]; + } + + /** + * Remove the head task from the priority queue. + */ + void removeMin() { + queue[1] = queue[size]; + queue[size--] = null; // Drop extra reference to prevent memory leak + fixDown(1); + } + + /** + * Removes the ith element from queue without regard for maintaining the + * heap invariant. Recall that queue is one-based, so 1 <= i <= size. + */ + void quickRemove(int i) { + assert i <= size; + + queue[i] = queue[size]; + queue[size--] = null; // Drop extra ref to prevent memory leak + } + + /** + * Sets the nextExecutionTime associated with the head task to the specified + * value, and adjusts priority queue accordingly. + */ + void rescheduleMin(long newTime) { + queue[1].nextExecutionTime = newTime; + fixDown(1); + } + + /** + * Returns true if the priority queue contains no elements. + */ + boolean isEmpty() { + return size == 0; + } + + /** + * Removes all elements from the priority queue. + */ + void clear() { + // Null out task references to prevent memory leak + for (int i = 1; i <= size; i++) + queue[i] = null; + + size = 0; + } + + /** + * Establishes the heap invariant (described above) assuming the heap + * satisfies the invariant except possibly for the leaf-node indexed by k + * (which may have a nextExecutionTime less than its parent's). + * + * This method functions by "promoting" queue[k] up the hierarchy (by + * swapping it with its parent) repeatedly until queue[k]'s + * nextExecutionTime is greater than or equal to that of its parent. + */ + private void fixUp(int k) { + while (k > 1) { + int j = k >> 1; + if (queue[j].nextExecutionTime <= queue[k].nextExecutionTime) + break; + TimedTask tmp = queue[j]; + queue[j] = queue[k]; + queue[k] = tmp; + k = j; + } + } + + /** + * Establishes the heap invariant (described above) in the subtree rooted at + * k, which is assumed to satisfy the heap invariant except possibly for + * node k itself (which may have a nextExecutionTime greater than its + * children's). + * + * This method functions by "demoting" queue[k] down the hierarchy (by + * swapping it with its smaller child) repeatedly until queue[k]'s + * nextExecutionTime is less than or equal to those of its children. + */ + private void fixDown(int k) { + int j; + while ((j = k << 1) <= size && j > 0) { + if (j < size + && queue[j].nextExecutionTime > queue[j + 1].nextExecutionTime) + j++; // j indexes smallest kid + if (queue[k].nextExecutionTime <= queue[j].nextExecutionTime) + break; + TimedTask tmp = queue[j]; + queue[j] = queue[k]; + queue[k] = tmp; + k = j; + } + } + + /** + * Establishes the heap invariant (described above) in the entire tree, + * assuming nothing about the order of the elements prior to the call. + */ + void heapify() { + for (int i = size / 2; i >= 1; i--) + fixDown(i); + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/java/hma/util/hdfs/TrashClient.java b/src/java/hma/util/hdfs/TrashClient.java new file mode 100644 index 0000000..5c9dcf7 --- /dev/null +++ b/src/java/hma/util/hdfs/TrashClient.java @@ -0,0 +1,88 @@ +package hma.util.hdfs; + +import hma.util.hdfs.protocol.DataNodeTrashProtocol; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; + +import java.io.IOException; +import java.net.InetSocketAddress; + +/** + * Created with IntelliJ IDEA. + * User: wangjian + * Date: 2013-11-07 + * Time: 22:44:00 + */ + +public class TrashClient { + public static final Log LOG = LogFactory.getLog(TrashClient.class); + public DataNodeTrashProtocol trashServer = null; + private org.apache.hadoop.conf.Configuration conf = null; + + public TrashClient(Configuration conf) throws IOException { + //super(conf); + this.conf = conf; + String hostname = this.conf.get(DataNodeTrashProtocol.DFS_TRASH_IPC_HOSTNAME_KEY, + null); + int port = this.conf.getInt(DataNodeTrashProtocol.DFS_TRASH_IPC_PORT_KEY, + 0); + if (null == hostname) { + LOG.error(DataNodeTrashProtocol.DFS_TRASH_IPC_HOSTNAME_KEY + " is not given in conf "); + } + if (0 == port) { + LOG.error(DataNodeTrashProtocol.DFS_TRASH_IPC_PORT_KEY + " is not given in conf"); + } + InetSocketAddress trashServerAddr = new InetSocketAddress(hostname, port); + try { + trashServer = (DataNodeTrashProtocol) RPC.getProxy(DataNodeTrashProtocol.class, + DataNodeTrashProtocol.versionID, trashServerAddr, conf); + } catch (Throwable t) { + trashServer = null; + } + } + + public long getTrashRetentionSecondsInternal() { + long retentionSeconds = Long.MIN_VALUE; + try { + retentionSeconds = trashServer.getTrashRetentionSeconds(); + } catch (Throwable t) { + LOG.error("failed to get retentionSeconds from trash server"); + } + return retentionSeconds; + } + + public void close() { + RPC.stopProxy(trashServer); + } + + public static long getTrashRetentionSeconds(Configuration conf) { + TrashClient trashClient = null; + try { + trashClient = new TrashClient(conf); + return trashClient.getTrashRetentionSecondsInternal(); + } catch (IOException e) { + e.printStackTrace(); + LOG.error("Failed to connect to trash Server"); + return Long.MIN_VALUE; + } + + } + + + public static void main(String[] args) throws IOException { + Configuration conf = new Configuration(); + String trashServer = args[0]; + String port = args[1]; + System.out.println(trashServer); + System.out.println(port); + conf.set(DataNodeTrashProtocol.DFS_TRASH_IPC_HOSTNAME_KEY, trashServer); + conf.set(DataNodeTrashProtocol.DFS_TRASH_IPC_PORT_KEY, port); + + System.out.println(TrashClient.getTrashRetentionSeconds(conf)); + + } + + +} diff --git a/src/java/hma/util/hdfs/protocol/DataNodeTrashProtocol.java b/src/java/hma/util/hdfs/protocol/DataNodeTrashProtocol.java new file mode 100644 index 0000000..71ad711 --- /dev/null +++ b/src/java/hma/util/hdfs/protocol/DataNodeTrashProtocol.java @@ -0,0 +1,18 @@ +package hma.util.hdfs.protocol; + +import org.apache.hadoop.ipc.VersionedProtocol; + +public interface DataNodeTrashProtocol extends VersionedProtocol { + public static final long versionID = 1L; + + public long getTrashRetentionSeconds(); + + public static final String DFS_TRASH_RETENTION_TIME_KEY = "dfs.trash.retention.time.second"; + public static final long DFS_TRASH_RETENTION_TIME_DEFAULT_VALUE = Long.MAX_VALUE; + public static final String DFS_TRASH_IPC_HOSTNAME_KEY = "dfs.trash.ipc.hostname"; + public static final String DFS_TRASH_IPC_HOSTNAME_DEFAULT_VALUE = "localhost"; + public static final String DFS_TRASH_IPC_PORT_KEY = "dfs.trash.ipc.port"; + public static final int DFS_TRASH_IPC_PORT_DEFAULT_VALUE = 22570; + public static final String DFS_TRASH_IPC_HANDLER_COUNT_KEY = "dfs.trash.ipc.handler.count"; + public static final int DFS_TRASH_IPC_HANDLER_COUNT_DEFAULT_VALUE = 50; +} diff --git a/src/java/hma/web/HMAJspHelper.java b/src/java/hma/web/HMAJspHelper.java new file mode 100644 index 0000000..daec994 --- /dev/null +++ b/src/java/hma/web/HMAJspHelper.java @@ -0,0 +1,1547 @@ +/** + * + */ +package hma.web; + +import hma.conf.Configuration; +import hma.monitor.collection.metric.CollectionMetricRecord; + +import hma.monitor.strategy.MonitorStrategyConfiguration; +import hma.monitor.strategy.MonitorStrategyPrototype; +import hma.util.HMALogger; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; + +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.sql.*; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.Date; + +/** + * @author guoyezhi + */ +public class HMAJspHelper { + + private String dbServer = null; + private String dbName = null; + + private String resultTableName = null; + private String infoTableName = null; + + private String collectionMetricTableName = null; + + /** + * DB Table for storing information about Unworking Datanodes + */ + private String extUDNTableName = null; + /** + * DB Table for storing information about User INode Quota Info + */ + private String extUserINodeQuotaInfoTableName = null; + /** + * DB Table for storing information about User INode Quota Info + */ + private String extUserSpaceQuotaInfoTableName = null; + /** + * DB Table for OOB monitor imports. + */ + private String extOOBMonitorImportsTableName = null; + + /** + * Inspection Table for daily inspection + */ + private String inspectionTable = null; + + private String userName = null; + private String password = null; + + private static Configuration clusterStatConf = + new Configuration(true, "hma-clusterhealth"); + private static MonitorStrategyConfiguration strategyConf = + new MonitorStrategyConfiguration(true); + private static Map strategyName2AliasMapping = null; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + + public HMAJspHelper() { + + Configuration hmaConf = new Configuration(true, "hma"); + + dbServer = hmaConf.get( + "monitor.data.db.server", "yf-2950-016.yf01.baidu.com"); + dbName = hmaConf.get( + "monitor.data.db.name", "hma"); + + resultTableName = hmaConf.get( + "monitor.result.data.db.table", "monitor_result"); + infoTableName = hmaConf.get( + "monitor.info.data.db.table", "monitor_info"); + collectionMetricTableName = hmaConf.get( + "monitor.collection.metric.db.table", + "monitor_cmetric"); + extUDNTableName = hmaConf.get( + "monitor.extension.unworkingdatanodes.info.db.table", + "monitor_ext_unworkingdatanodes"); + extUserINodeQuotaInfoTableName = hmaConf.get( + "monitor.extension.user.inode.quota.db.table", + "monitor_ext_userinodequota"); + extUserSpaceQuotaInfoTableName = hmaConf.get( + "monitor.extension.user.space.quota.db.table", + "monitor_ext_userspacequota"); + extOOBMonitorImportsTableName = hmaConf.get( + "monitor.extension.oob.monitor.imports.table", + "monitor_ext_oobmonimports"); + inspectionTable = hmaConf.get( + "monitor.inspection.data.db.table", "monitor_inspection"); + + userName = hmaConf.get( + "monitor.db.user.name", "root"); + password = hmaConf.get( + "monitor.db.user.pass", "hmainit"); + + } + + + public static synchronized boolean isMonitorStrategyTransient( + String strategyName) { + String str = clusterStatConf.get(strategyName, "permanent"); + if (str.toLowerCase().equals("transient")) + return true; + return false; + } + + public static synchronized HashMap + getAllMonitorStrategyPrototypes() { + return strategyConf.getAllPrototypes(); + } + + public static synchronized MonitorStrategyPrototype + getMonitorStrategyPrototype(String strategyName) { + return strategyConf.getAllPrototypes().get(strategyName); + } + + public static synchronized String getMonitorStrategyAlias(String strategyName) { + if (strategyName2AliasMapping == null) { + strategyName2AliasMapping = new HashMap(); + HashMap entries = + strategyConf.getAllPrototypes(); + Iterator iter = entries.keySet().iterator(); + while (iter.hasNext()) { + String name = iter.next(); + String alias = entries.get(name).getAlias(); + strategyName2AliasMapping.put(name, alias); + } + } + return strategyName2AliasMapping.get(strategyName); + } + + public static class StrategyComparator + implements Comparator { + + @Override + public int compare(MonitorStrategyPrototype arg0, + MonitorStrategyPrototype arg1) { + if(arg0 == null || arg1 == null ){ + return 0; + } + System.err.println(arg0.getName() + " vs. " + arg1.getName()); + int level0 = arg0.getAlarmLevel(); + int level1 = arg1.getAlarmLevel(); + if (level0 > level1) { + return -1; + } else if (level0 < level1) { + return 1; + } + String name0 = arg0.getName(); + String alias0 = arg0.getAlias(); + if (alias0 != null) name0 = alias0; + String name1 = arg1.getName(); + String alias1 = arg1.getAlias(); + if (alias1 != null) name1 = alias1; + return name0.compareTo(name1); + } + + } + + + private Connection createDBConnection() throws SQLException { + + System.out.println( + "jdbc:mysql://" + dbServer + "/" + dbName + + "/" + userName + "@" + password); + + return DriverManager.getConnection( + "jdbc:mysql://" + dbServer + "/" + dbName, + userName, + password); + + /* + return DriverManager.getConnection( + "jdbc:mysql://" + dbServer + "/" + dbName + + "?useUnicode=true&characterEncoding=utf8", + userName, + password); + */ + } + + + public static final int DetectionResult_NODATA = 0x00; + public static final int DetectionResult_NORMAL = 0x01; + public static final int DetectionResult_HASINFO = 0x02; + public static final int DetectionResult_NOTICEABLE = 0x03; + public static final int DetectionResult_ANORMLOUS = 0x04; + + public synchronized Map checkDetectionResults( + String clusterName, + String begStr, + String endStr) throws SQLException { + + Map checkResult = new HashMap(); + + Connection dbConn = createDBConnection(); + + Iterator> iter = + clusterStatConf.iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String strategyName = entry.getKey(); + boolean permanent = + entry.getValue().toLowerCase().equals("permanent"); + + PreparedStatement pstat = null; + if (permanent == false) { + pstat = dbConn.prepareStatement( + "SELECT detection_timestamp, strategy_level, detection_result " + + "FROM " + resultTableName + " " + + "WHERE cluster_name=? " + + "AND strategy_name=? " + + "AND detection_timestamp>? " + + "AND detection_timestamp? " + + "AND detection_timestamp<=? " + + "ORDER BY detection_timestamp ASC"); + pstat.setString(1, clusterName); + pstat.setString(2, strategyName); + pstat.setString(3, begStr); + pstat.setString(4, endStr); + ResultSet res = pstat.executeQuery(); + + boolean anomalous = false; + boolean notice = true; + boolean info = true; + + int cnt = 0; + for (; res.next(); cnt++) { + anomalous = anomalous || res.getBoolean("detection_result"); + if (res.getBoolean("detection_result")) { + String level = res.getString("strategy_level").toUpperCase(); + if (level.equals("EMERG") || level.equals("CRITICAL") + || level.equals("WARN") || level.equals("WARNING")) { + notice = false; + info = false; + } else if (level.equals("NOTICE")) { + info = false; + } + } + } + if (cnt > 0) { + if (anomalous) { + if (info) { + checkResult.put(strategyName, DetectionResult_HASINFO); + } else if (notice) { + checkResult.put(strategyName, DetectionResult_NOTICEABLE); + } else { + checkResult.put(strategyName, DetectionResult_ANORMLOUS); + } + } else { + checkResult.put(strategyName, DetectionResult_NORMAL); + } + } else { + checkResult.put(strategyName, DetectionResult_NODATA); + } + } + } + + return checkResult; + } + + public synchronized Map checkOOBMonitorImportResults( + String clusterName, + String begStr, + String endStr) throws SQLException { + + Map checkResult = new HashMap(); + + Connection dbConn = createDBConnection(); + + PreparedStatement pstat = null; + pstat = dbConn.prepareStatement( + "SELECT DISTINCT oob_monitor_name " + + "FROM " + extOOBMonitorImportsTableName + " " + + "WHERE cluster_name=? " + + "AND detection_timestamp>? " + + "AND detection_timestamp<=? " + + "ORDER BY oob_monitor_name"); + pstat.setString(1, clusterName); + pstat.setString(2, begStr); + pstat.setString(3, endStr); + ResultSet res = pstat.executeQuery(); + while (res.next()) { + checkResult.put(res.getString("oob_monitor_name"), + DetectionResult_NORMAL); + } + pstat.close(); + + for (String oobMonitorName : checkResult.keySet()) { + pstat = dbConn.prepareStatement( + "SELECT detection_timestamp, oob_monitor_result " + + "FROM " + extOOBMonitorImportsTableName + " " + + "WHERE cluster_name=? " + + "AND oob_monitor_name=? " + + "AND detection_timestamp>? " + + "AND detection_timestamp<=? " + + "ORDER BY detection_timestamp ASC"); + pstat.setString(1, clusterName); + pstat.setString(2, oobMonitorName); + pstat.setString(3, begStr); + pstat.setString(4, endStr); + res = pstat.executeQuery(); + + boolean anomalous = false; + int cnt = 0; + for (; res.next(); cnt++) { + anomalous = anomalous || res.getBoolean("oob_monitor_result"); + } + if (cnt > 0) { + if (anomalous) { + checkResult.put(oobMonitorName, DetectionResult_ANORMLOUS); + } else { + checkResult.put(oobMonitorName, DetectionResult_NORMAL); + } + } else { + checkResult.put(oobMonitorName, DetectionResult_NODATA); + } + pstat.close(); + } + + return checkResult; + } + + + public static class MonitorStrategyAnomaly { + + private String clusterName = null; + + private String strategyName = null; + + private Long detectionTimestamp = null; + + private String infoLevel = null; + + private String infoContent = null; + + public MonitorStrategyAnomaly( + Long detectionTimestamp, + String infoLevel, + String infoContent) { + this.detectionTimestamp = detectionTimestamp; + this.infoLevel = infoLevel; + this.infoContent = infoContent; + } + + public String getClusterName() { + return clusterName; + } + + public String getStrategyName() { + return strategyName; + } + + public Long getDetectionTimestamp() { + return detectionTimestamp; + } + + public String getInfoLevel() { + return infoLevel; + } + + public String getInfoContent() { + return infoContent; + } + } + + public synchronized MonitorStrategyAnomaly getLastRecentMonitorStrategyAnomaly( + String clusterName, + String strategyName) throws SQLException { + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT detection_timestamp, info_level, info_content " + + "FROM " + infoTableName + " " + + "WHERE cluster_name=? " + + "AND strategy_name=? " + + "ORDER BY detection_timestamp DESC limit 1"); + pstat.setString(1, clusterName); + pstat.setString(2, strategyName); + ResultSet res = pstat.executeQuery(); + if (res.next()) { + Timestamp detectionTimestamp = res.getTimestamp("detection_timestamp"); + String infoLevel = res.getString("info_level"); + String infoContent = res.getString("info_content"); + return new MonitorStrategyAnomaly( + detectionTimestamp.getTime(), infoLevel, infoContent); + } + return null; + } + + public synchronized NavigableMap + getMonitorStrategyAnomaliesWithinPeriod( + String clusterName, + String strategyName, + long begginning, long end) throws SQLException { + + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT detection_timestamp, info_level, info_content " + + "FROM " + infoTableName + " " + + "WHERE cluster_name=? " + + "AND strategy_name=? " + + "AND detection_timestamp>? " + + "AND detection_timestamp<=? " + + "ORDER BY detection_timestamp"); + pstat.setString(1, clusterName); + pstat.setString(2, strategyName); + pstat.setString(3, dateFormat.format(new Date(begginning))); + pstat.setString(4, dateFormat.format(new Date(end))); + ResultSet res = pstat.executeQuery(); + + NavigableMap anomalies = + new TreeMap(); + + while (res.next()) { + Timestamp detectionTimestamp = res.getTimestamp("detection_timestamp"); + String infoLevel = res.getString("info_level"); + String infoContent = res.getString("info_content"); + anomalies.put(detectionTimestamp.getTime(), + new MonitorStrategyAnomaly( + detectionTimestamp.getTime(), + infoLevel, infoContent)); + } + + return anomalies; + } + + public synchronized NavigableMap + getOOBMonitorLogsWithinPeriod( + String clusterName, + String oobMonitorName, + long begginning, + long end) throws SQLException { + + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT detection_timestamp, oob_monitor_log " + + "FROM " + extOOBMonitorImportsTableName + " " + + "WHERE cluster_name=? " + + "AND oob_monitor_name=? " + + "AND oob_monitor_result=? " + + "AND detection_timestamp>? " + + "AND detection_timestamp<=? " + + "ORDER BY detection_timestamp"); + pstat.setString(1, clusterName); + pstat.setString(2, oobMonitorName); + pstat.setBoolean(3, true); + pstat.setString(4, dateFormat.format(new Date(begginning))); + pstat.setString(5, dateFormat.format(new Date(end))); + ResultSet res = pstat.executeQuery(); + + NavigableMap oobMonitorLogs = new TreeMap(); + + while (res.next()) { + Timestamp detectionTimestamp = res.getTimestamp("detection_timestamp"); + String log = res.getString("oob_monitor_log"); + oobMonitorLogs.put(detectionTimestamp.getTime(), log); + } + + return oobMonitorLogs; + } + + public static void main(String[] args) throws SQLException, UnsupportedEncodingException { + + Connection conn = DriverManager.getConnection( + "jdbc:mysql://yf-2950-hma-db.yf01.baidu.com/hma?useUnicode=true&characterEncoding=utf8", + "root", "hmainit"); + PreparedStatement pstat = conn.prepareStatement( + "SELECT detection_timestamp, info_level, info_content " + + "FROM monitor_info " + + "WHERE cluster_name=? " + + "AND strategy_name=? " + + "AND detection_timestamp>? " + + "AND detection_timestamp<=? " + + "ORDER BY detection_timestamp LIMIT 1"); + pstat.setString(1, "ECOM-OFF"); + pstat.setString(2, "HDFS_RsyncLastMinuteFsimage_MON"); + pstat.setString(3, "2011-01-11 22:38:10"); + pstat.setString(4, "2011-01-11 22:58:10"); + ResultSet res = pstat.executeQuery(); + if (res.next()) { + Timestamp detectionTimestamp = res.getTimestamp("detection_timestamp"); + String infoLevel = res.getString("info_level"); + String infoContent = res.getString("info_content"); + System.out.println(detectionTimestamp + " : " + infoLevel + " : " + infoContent); + byte[] bytes = res.getBytes("info_content"); + for (int i = 0; i < bytes.length; i++) { + System.out.print(Integer.toHexString(bytes[i]) + " "); + } + System.out.println(); + System.out.println(new String(bytes, "utf-8")); + } + /* + String str = "监控项 HDFS_RsyncLastMinuteFsimage_MON 数据采集超时 777 秒\n"; + byte[] bytes2 = str.getBytes("utf-8"); + for (int i = 0; i < bytes2.length; i++) { + System.out.print(Integer.toHexString(bytes2[i]) + " "); + } + System.out.println(); + + bytes2 = str.getBytes("utf-16"); + for (int i = 0; i < bytes2.length; i++) { + System.out.print(Integer.toHexString(bytes2[i]) + " "); + } + */ + + HMALogger logger = new HMALogger("hmalogger.txt"); + + /* + pstat = conn.prepareStatement("INSERT INTO monitor_info " + + "VALUES(\"Test\", \"Foo_MON\", \"2011-01-12 22:40:50\", \"WARN\", \"测试\")"); + System.out.println(pstat.toString()); + pstat.execute(); + */ + + pstat = conn.prepareStatement( + "SELECT detection_timestamp, info_level, info_content " + + "FROM monitor_info WHERE cluster_name=\"ECOM-OFF\" " + + "ORDER BY detection_timestamp DESC LIMIT 1"); + res = pstat.executeQuery(); + if (res.next()) { + Timestamp detectionTimestamp = res.getTimestamp("detection_timestamp"); + String infoLevel = res.getString("info_level"); + String infoContent = res.getString("info_content"); + System.out.println(detectionTimestamp + " : " + infoLevel + " : " + infoContent); + logger.log(detectionTimestamp + " : " + infoLevel + " : " + infoContent); + + Blob bl = res.getBlob("info_content"); + byte[] bytes = bl.getBytes(1, (int) bl.length()); + for (int i = 0; i < bytes.length; i++) { + System.out.print(Integer.toHexString(bytes[i]) + " "); + } + System.out.println(); + System.out.println(new String(bytes)); + System.out.println(new String(bytes, "utf-8")); + } + } + + + /* + * The following fields & methods are for `unworkingdatanodes.jsp' + */ + public static class DatanodeInfoComparator implements Comparator { + + @Override + public int compare(DatanodeInfo arg0, DatanodeInfo arg1) { + long timestamp0 = arg0.getLastUpdate(); + long timestamp1 = arg1.getLastUpdate(); + if (timestamp0 > timestamp1) { + return -1; + } else if (timestamp0 < timestamp1) { + return 1; + } + String name0 = arg0.getHostName(); + String name1 = arg1.getHostName(); + return name0.compareTo(name1); + } + + } + + /* public synchronized NavigableMap> getClusterUnworkingDatanodeInfo( + String clusterName) throws SQLException { + + TreeMap> monitorInfo = + new TreeMap>(); + + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT * FROM " + extUDNTableName + " " + + "WHERE cluster_name=? " + + "ORDER BY detection_timestamp DESC " + + "LIMIT 1"); + pstat.setString(1, clusterName); + ResultSet res = pstat.executeQuery(); + + if (res.next()) { + + Timestamp detectionTimestamp = res.getTimestamp("detection_timestamp"); + Blob blob = res.getBlob("serial_info"); + DataInputStream dataIS = + new DataInputStream(blob.getBinaryStream()); + try { + List unworkingList = new ArrayList(); + + int recreateLength = dataIS.readInt(); + for (int i = 0; i < recreateLength; i++) { + DatanodeInfo di = new DatanodeInfo(); + di.readFields(dataIS); + unworkingList.add(di); + } + + monitorInfo.put(detectionTimestamp.getTime(), unworkingList); + + } catch (IOException e) { + e.printStackTrace(); + return null; + } finally { + try { + dataIS.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + pstat.close(); + dbConn.close(); + + return monitorInfo; + }*/ + + + /*public synchronized int getClusterUnworkingDatanodeNum(*/ + /* String clusterName, long starttime, long endtime) throws SQLException {*/ +/* +*/ + +/* +*/ + + /* Connection dbConn = createDBConnection();*/ + /* PreparedStatement pstat = dbConn.prepareStatement(*/ + /* "SELECT * FROM " + extUDNTableName + " " +*/ + /* "WHERE cluster_name=? " +*/ + /* "ORDER BY detection_timestamp DESC " +*/ + /* "LIMIT 1");*/ + /* pstat.setString(1, clusterName);*/ + /* ResultSet res = pstat.executeQuery();*/ + /* int count = 0;*/ + /* if (res.next()) {*/ +/* +*/ + +/* +*/ + + /* Blob blob = res.getBlob("serial_info");*/ + /* DataInputStream dataIS =*/ + /* new DataInputStream(blob.getBinaryStream());*/ + /* try {*/ +/* +*/ + +/* +*/ + + /* int recreateLength = dataIS.readInt();*/ + /* for (int i = 0; i < recreateLength; i++) {*/ + /* DatanodeInfo di = new DatanodeInfo();*/ + /* di.readFields(dataIS);*/ + /* //unworkingList.add(di);*/ + /* long time = di.getLastUpdate();*/ + /* if (time >= starttime && time <= endtime && !di.isDecommissioned()) {*/ + /* count++;*/ + /* }*/ + /* }*/ + /* } catch (IOException e) {*/ + /* e.printStackTrace();*/ + /* return count;*/ + /* } finally {*/ + /* try {*/ + /* dataIS.close();*/ + /* } catch (IOException e) {*/ + /* e.printStackTrace();*/ + /* }*/ + /* }*/ + /* }*/ +/* +*/ + + /* pstat.close();*/ + /* dbConn.close();*/ +/* +*/ + + /* return count;*/ + /*}*/ + + + /* + * The following fields & methods are for `userstatus.jsp' + */ + public static class UserINodeQuotaInfo { + + private String userName = null; + + private String userAppPath = null; + + private long inodeQuota = -1L; + + private long inodeFileCount = 0L; + + private long inodeDirectoryCount = 0L; + + private Date detectionTimestamp = null; + + private double inodeAlarmThreshold = 80.0; + + public UserINodeQuotaInfo( + String userName, + String userAppPath, + long inodeQuota, + long inodeFileCount, + long inodeDirectoryCount, + Date detectionTimestamp, + double inodeAlarmThreshold) { + this.userName = userName; + this.userAppPath = userAppPath; + this.inodeQuota = inodeQuota; + this.inodeFileCount = inodeFileCount; + this.inodeDirectoryCount = inodeDirectoryCount; + this.detectionTimestamp = detectionTimestamp; + this.inodeAlarmThreshold = inodeAlarmThreshold; + } + + public String getUserName() { + return userName; + } + + public String getUserAppPath() { + return userAppPath; + } + + public long getInodeQuota() { + return inodeQuota; + } + + public long getInodeFileCount() { + return inodeFileCount; + } + + public long getInodeDirectoryCount() { + return inodeDirectoryCount; + } + + public Date getDetectionTimestamp() { + return detectionTimestamp; + } + + public double getInodeAlarmThreshold() { + return inodeAlarmThreshold; + } + + } + + public static class UserINodeQuotaInfoComparator + implements Comparator { + + @Override + public int compare(UserINodeQuotaInfo info0, UserINodeQuotaInfo info1) { + + String userName0 = info0.getUserName(); + String userName1 = info1.getUserName(); + + if (userName0.compareTo(userName1) != 0) + return userName0.compareTo(userName1); + + String userAppPath0 = info0.getUserAppPath(); + String userAppPath1 = info1.getUserAppPath(); + + return userAppPath0.compareTo(userAppPath1); + } + + } + + /*public synchronized NavigableSet + getAllUserINodeQuotaInfo(String clusterName) throws SQLException { + + NavigableSet monitorInfo = + new TreeSet(new UserINodeQuotaInfoComparator()); + + Configuration hmaConf = new Configuration(true, "hma"); + String confBaseDirPath = hmaConf.get( + "hma.conf.base.dir", "/home/monitor/hma-v1.1/hma/conf/"); + if (!confBaseDirPath.endsWith("/")) { + confBaseDirPath = confBaseDirPath + "/"; + } + String defaultConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-default.xml"; + String siteConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-site.xml"; + File defaultConfFile = new File(defaultConfFilePath); + File siteConfFile = new File(siteConfFilePath); + if (!defaultConfFile.exists() && !siteConfFile.exists()) { + throw new RuntimeException(); + } + + UserQuotaConfiguration quotaConf = + new UserQuotaConfiguration(false); + if (defaultConfFile.exists()) { + quotaConf.addConfigurationResource(defaultConfFile); + } + if (siteConfFile.exists()) { + quotaConf.addConfigurationResource(siteConfFile); + } + NavigableSet confEntries = + quotaConf.loadEntries(); + + Connection dbConn = createDBConnection(); + for (UserQuotaConfigurationEntry entry : confEntries) { + + System.out.println(entry); + + String userName = entry.getName(); + String userAppPath = entry.getAppPath(); + long inodeQuota = entry.getInodeQuota(); + double inodeAlarmThreshold = entry.getInodeAlarmThreshold(); + + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT * FROM " + extUserINodeQuotaInfoTableName + " " + + "WHERE cluster_name=? " + + "AND user_name=? " + + "AND user_app_path=? " + + "AND inode_is_sub_path=0 " + + "ORDER BY detection_timestamp DESC " + + "LIMIT 1"); + pstat.setString(1, clusterName.toUpperCase()); + pstat.setString(2, userName); + pstat.setString(3, userAppPath); + ResultSet res = pstat.executeQuery(); + + if (res.next()) { + + Date detectionTimestamp = + new Date(res.getTimestamp("detection_timestamp").getTime()); + long inodeFileCount = res.getLong("inode_file_count"); + long inodeDirectoryCount = res.getLong("inode_directory_count"); + + monitorInfo.add(new UserINodeQuotaInfo( + userName, + userAppPath, + inodeQuota, + inodeFileCount, + inodeDirectoryCount, + detectionTimestamp, + inodeAlarmThreshold)); + + } else { + monitorInfo.add(new UserINodeQuotaInfo( + userName, + userAppPath, + inodeQuota, + -1, + -1, + new Date(0), + inodeAlarmThreshold)); + } + + pstat.close(); + } + + dbConn.close(); + + return monitorInfo; + } */ + + /* + public synchronized NavigableMap getNDaysUserINodeQuotaInfoOfTargetPath( + String clusterName, String userName, String userAppPath, long lastDetectedAt, int days) + throws SQLException { + + NavigableMap monitorInfo = + new TreeMap(); + + Configuration hmaConf = new Configuration(true, "hma"); + String confBaseDirPath = hmaConf.get( + "hma.conf.base.dir", "/home/monitor/hma-v1.1/hma/conf/"); + if (!confBaseDirPath.endsWith("/")) { + confBaseDirPath = confBaseDirPath + "/"; + } + String defaultConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-default.xml"; + String siteConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-site.xml"; + File defaultConfFile = new File(defaultConfFilePath); + File siteConfFile = new File(siteConfFilePath); + if (!defaultConfFile.exists() && !siteConfFile.exists()) { + throw new RuntimeException(); + } + + UserQuotaConfiguration quotaConf = + new UserQuotaConfiguration(false); + if (defaultConfFile.exists()) { + quotaConf.addConfigurationResource(defaultConfFile); + } + if (siteConfFile.exists()) { + quotaConf.addConfigurationResource(siteConfFile); + } + NavigableSet confEntries = quotaConf.loadEntries(); + + UserQuotaConfigurationEntry targetConfEntry = null; + for (UserQuotaConfigurationEntry confEntry : confEntries) { + if (confEntry.getName().equals(userName) && + confEntry.getAppPath().equals(userAppPath)) { + targetConfEntry = confEntry; + break; + } + } + + long inodeQuota = targetConfEntry.getInodeQuota(); + double alarmThreshold = targetConfEntry.getInodeAlarmThreshold(); + + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT * FROM " + extUserINodeQuotaInfoTableName + " " + + "WHERE cluster_name=? " + + "AND user_name=? " + + "AND user_app_path=? " + + "AND detection_timestamp<=? " + + "AND detection_timestamp>? " + + "AND inode_is_sub_path=0 " + + "ORDER BY detection_timestamp"); + pstat.setString(1, clusterName.toUpperCase()); + pstat.setString(2, userName); + pstat.setString(3, userAppPath); + pstat.setString(4, dateFormat.format(new Date(lastDetectedAt))); + pstat.setString(5, + dateFormat.format(new Date(lastDetectedAt - ((long) days) * 24 * 3600 * 1000))); + + ResultSet res = pstat.executeQuery(); + while (res.next()) { + + Date detectionTimestamp = + new Date(res.getTimestamp("detection_timestamp").getTime()); + String hdfsPath = res.getString("user_app_path"); + long inodeFileCount = res.getLong("inode_file_count"); + long inodeDirectoryCount = res.getLong("inode_directory_count"); + + monitorInfo.put(detectionTimestamp, + new UserINodeQuotaInfo( + userName, + hdfsPath, + inodeQuota, + inodeFileCount, + inodeDirectoryCount, + detectionTimestamp, + alarmThreshold)); + } + + pstat.close(); + dbConn.close(); + + return monitorInfo; + } + + public synchronized NavigableSet getUserINodeQuotaTotalInfoOfTargetPath( + String clusterName, String userName, String userAppPath, long detectedAt) + throws SQLException { + + NavigableSet monitorInfo = + new TreeSet(new UserINodeQuotaInfoComparator()); + + Configuration hmaConf = new Configuration(true, "hma"); + String confBaseDirPath = hmaConf.get( + "hma.conf.base.dir", "/home/monitor/hma-v1.1/hma/conf/"); + if (!confBaseDirPath.endsWith("/")) { + confBaseDirPath = confBaseDirPath + "/"; + } + String defaultConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-default.xml"; + String siteConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-site.xml"; + File defaultConfFile = new File(defaultConfFilePath); + File siteConfFile = new File(siteConfFilePath); + if (!defaultConfFile.exists() && !siteConfFile.exists()) { + throw new RuntimeException(); + } + + UserQuotaConfiguration quotaConf = + new UserQuotaConfiguration(false); + if (defaultConfFile.exists()) { + quotaConf.addConfigurationResource(defaultConfFile); + } + if (siteConfFile.exists()) { + quotaConf.addConfigurationResource(siteConfFile); + } + NavigableSet confEntries = quotaConf.loadEntries(); + + UserQuotaConfigurationEntry targetConfEntry = null; + for (UserQuotaConfigurationEntry confEntry : confEntries) { + if (confEntry.getName().equals(userName) && + confEntry.getAppPath().equals(userAppPath)) { + targetConfEntry = confEntry; + break; + } + } + + long inodeQuota = targetConfEntry.getInodeQuota(); + double alarmThreshold = targetConfEntry.getInodeAlarmThreshold(); + + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT * FROM " + extUserINodeQuotaInfoTableName + " " + + "WHERE cluster_name=? " + + "AND user_name=? " + + "AND detection_timestamp=? "); + pstat.setString(1, clusterName.toUpperCase()); + pstat.setString(2, userName); + pstat.setString(3, dateFormat.format(new Date(detectedAt))); + + ResultSet res = pstat.executeQuery(); + while (res.next()) { + + Date detectionTimestamp = + new Date(res.getTimestamp("detection_timestamp").getTime()); + String hdfsPath = res.getString("user_app_path"); + long inodeFileCount = res.getLong("inode_file_count"); + long inodeDirectoryCount = res.getLong("inode_directory_count"); + + monitorInfo.add(new UserINodeQuotaInfo( + userName, + hdfsPath, + inodeQuota, + inodeFileCount, + inodeDirectoryCount, + detectionTimestamp, + alarmThreshold)); + } + + pstat.close(); + dbConn.close(); + + return monitorInfo; + + } + + + public static class UserSpaceQuotaInfo { + + private String userName = null; + + private String userAppPath = null; + + private long spaceQuota = -1L; + + private long spaceConsumed = 0L; + + private Date detectionTimestamp = null; + + private double spaceAlarmThreshold = 80.0; + + public UserSpaceQuotaInfo( + String userName, + String userAppPath, + long spaceQuota, + long spaceConsumed, + Date detectionTimestamp, + double spaceAlarmThreshold) { + this.userName = userName; + this.userAppPath = userAppPath; + this.spaceQuota = spaceQuota; + this.spaceConsumed = spaceConsumed; + this.detectionTimestamp = detectionTimestamp; + this.spaceAlarmThreshold = spaceAlarmThreshold; + } + + public String getUserName() { + return userName; + } + + public String getUserAppPath() { + return userAppPath; + } + + public long getSpaceQuota() { + return spaceQuota; + } + + public long getSpaceConsumed() { + return spaceConsumed; + } + + public Date getDetectionTimestamp() { + return detectionTimestamp; + } + + public double getSpaceAlarmThreshold() { + return spaceAlarmThreshold; + } + + } + + public static class UserSpaceQuotaInfoComparator + implements Comparator { + + @Override + public int compare(UserSpaceQuotaInfo info0, UserSpaceQuotaInfo info1) { + + String userName0 = info0.getUserName(); + String userName1 = info1.getUserName(); + + if (userName0.compareTo(userName1) != 0) + return userName0.compareTo(userName1); + + String userAppPath0 = info0.getUserAppPath(); + String userAppPath1 = info1.getUserAppPath(); + + return userAppPath0.compareTo(userAppPath1); + } + + } + + public synchronized NavigableSet + getAllUserSpaceQuotaInfo(String clusterName) throws SQLException { + + NavigableSet monitorInfo = + new TreeSet(new UserSpaceQuotaInfoComparator()); + + Configuration hmaConf = new Configuration(true, "hma"); + String confBaseDirPath = hmaConf.get( + "hma.conf.base.dir", "/home/monitor/hma-v1.1/hma/conf/"); + if (!confBaseDirPath.endsWith("/")) { + confBaseDirPath = confBaseDirPath + "/"; + } + String defaultConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-default.xml"; + String siteConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-site.xml"; + File defaultConfFile = new File(defaultConfFilePath); + File siteConfFile = new File(siteConfFilePath); + if (!defaultConfFile.exists() && !siteConfFile.exists()) { + throw new RuntimeException(); + } + + UserQuotaConfiguration quotaConf = + new UserQuotaConfiguration(false); + if (defaultConfFile.exists()) { + quotaConf.addConfigurationResource(defaultConfFile); + } + if (siteConfFile.exists()) { + quotaConf.addConfigurationResource(siteConfFile); + } + NavigableSet confEntries = + quotaConf.loadEntries(); + + Connection dbConn = createDBConnection(); + for (UserQuotaConfigurationEntry entry : confEntries) { + + System.out.println(entry); + + String userName = entry.getName(); + String userAppPath = entry.getAppPath(); + long spaceQuota = entry.getSpaceQuota(); + double spaceAlarmThreshold = entry.getSpaceAlarmThreshold(); + + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT * FROM " + extUserSpaceQuotaInfoTableName + " " + + "WHERE cluster_name=? " + + "AND user_name=? " + + "AND user_app_path=? " + + "AND inode_is_sub_path=0 " + + "ORDER BY detection_timestamp DESC " + + "LIMIT 1"); + pstat.setString(1, clusterName.toUpperCase()); + pstat.setString(2, userName); + pstat.setString(3, userAppPath); + ResultSet res = pstat.executeQuery(); + + if (res.next()) { + + Date detectionTimestamp = + new Date(res.getTimestamp("detection_timestamp").getTime()); + long spaceConsumed = res.getLong("space_consumed"); + + monitorInfo.add(new UserSpaceQuotaInfo( + userName, + userAppPath, + spaceQuota, + spaceConsumed, + detectionTimestamp, + spaceAlarmThreshold)); + + } else { + monitorInfo.add(new UserSpaceQuotaInfo( + userName, + userAppPath, + spaceQuota, + -1, + new Date(0), + spaceAlarmThreshold)); + } + + pstat.close(); + } + + dbConn.close(); + + return monitorInfo; + } + + public synchronized NavigableMap getNDaysUserSpaceQuotaInfoOfTargetPath( + String clusterName, String userName, String userAppPath, long lastDetectedAt, int days) + throws SQLException { + + NavigableMap monitorInfo = + new TreeMap(); + + Configuration hmaConf = new Configuration(true, "hma"); + String confBaseDirPath = hmaConf.get( + "hma.conf.base.dir", "/home/monitor/hma-v1.1/hma/conf/"); + if (!confBaseDirPath.endsWith("/")) { + confBaseDirPath = confBaseDirPath + "/"; + } + String defaultConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-default.xml"; + String siteConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-site.xml"; + File defaultConfFile = new File(defaultConfFilePath); + File siteConfFile = new File(siteConfFilePath); + if (!defaultConfFile.exists() && !siteConfFile.exists()) { + throw new RuntimeException(); + } + + UserQuotaConfiguration quotaConf = + new UserQuotaConfiguration(false); + if (defaultConfFile.exists()) { + quotaConf.addConfigurationResource(defaultConfFile); + } + if (siteConfFile.exists()) { + quotaConf.addConfigurationResource(siteConfFile); + } + NavigableSet confEntries = quotaConf.loadEntries(); + + UserQuotaConfigurationEntry targetConfEntry = null; + for (UserQuotaConfigurationEntry confEntry : confEntries) { + if (confEntry.getName().equals(userName) && + confEntry.getAppPath().equals(userAppPath)) { + targetConfEntry = confEntry; + break; + } + } + + long spaceQuota = targetConfEntry.getSpaceQuota(); + double alarmThreshold = targetConfEntry.getSpaceAlarmThreshold(); + + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT * FROM " + extUserSpaceQuotaInfoTableName + " " + + "WHERE cluster_name=? " + + "AND user_name=? " + + "AND user_app_path=? " + + "AND detection_timestamp<=? " + + "AND detection_timestamp>? " + + "AND inode_is_sub_path=0 " + + "ORDER BY detection_timestamp"); + pstat.setString(1, clusterName.toUpperCase()); + pstat.setString(2, userName); + pstat.setString(3, userAppPath); + pstat.setString(4, dateFormat.format(new Date(lastDetectedAt))); + pstat.setString(5, + dateFormat.format(new Date(lastDetectedAt - ((long) days) * 24 * 3600 * 1000))); + + ResultSet res = pstat.executeQuery(); + while (res.next()) { + + Date detectionTimestamp = + new Date(res.getTimestamp("detection_timestamp").getTime()); + String hdfsPath = res.getString("user_app_path"); + long spaceConsumed = res.getLong("space_consumed"); + + monitorInfo.put(detectionTimestamp, + new UserSpaceQuotaInfo( + userName, + hdfsPath, + spaceQuota, + spaceConsumed, + detectionTimestamp, + alarmThreshold)); + } + + pstat.close(); + dbConn.close(); + + return monitorInfo; + } + + public synchronized NavigableSet getUserSpaceQuotaTotalInfoOfTargetPath( + String clusterName, String userName, String userAppPath, long detectedAt) + throws SQLException { + + NavigableSet monitorInfo = + new TreeSet(new UserSpaceQuotaInfoComparator()); + + Configuration hmaConf = new Configuration(true, "hma"); + String confBaseDirPath = hmaConf.get( + "hma.conf.base.dir", "/home/monitor/hma-v1.1/hma/conf/"); + if (!confBaseDirPath.endsWith("/")) { + confBaseDirPath = confBaseDirPath + "/"; + } + String defaultConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-default.xml"; + String siteConfFilePath = + confBaseDirPath + clusterName.toUpperCase() + + "/hma-user-quota-site.xml"; + File defaultConfFile = new File(defaultConfFilePath); + File siteConfFile = new File(siteConfFilePath); + if (!defaultConfFile.exists() && !siteConfFile.exists()) { + throw new RuntimeException(); + } + + UserQuotaConfiguration quotaConf = + new UserQuotaConfiguration(false); + if (defaultConfFile.exists()) { + quotaConf.addConfigurationResource(defaultConfFile); + } + if (siteConfFile.exists()) { + quotaConf.addConfigurationResource(siteConfFile); + } + NavigableSet confEntries = quotaConf.loadEntries(); + + UserQuotaConfigurationEntry targetConfEntry = null; + for (UserQuotaConfigurationEntry confEntry : confEntries) { + if (confEntry.getName().equals(userName) && + confEntry.getAppPath().equals(userAppPath)) { + targetConfEntry = confEntry; + break; + } + } + + long spaceQuota = targetConfEntry.getSpaceQuota(); + double alarmThreshold = targetConfEntry.getSpaceAlarmThreshold(); + + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT * FROM " + extUserSpaceQuotaInfoTableName + " " + + "WHERE cluster_name=? " + + "AND user_name=? " + + "AND detection_timestamp=? "); + pstat.setString(1, clusterName.toUpperCase()); + pstat.setString(2, userName); + pstat.setString(3, dateFormat.format(new Date(detectedAt))); + + ResultSet res = pstat.executeQuery(); + while (res.next()) { + + Date detectionTimestamp = + new Date(res.getTimestamp("detection_timestamp").getTime()); + String hdfsPath = res.getString("user_app_path"); + long spaceConsumed = res.getLong("space_consumed"); + + monitorInfo.add(new UserSpaceQuotaInfo( + userName, + hdfsPath, + spaceQuota, + spaceConsumed, + detectionTimestamp, + alarmThreshold)); + } + + pstat.close(); + dbConn.close(); + + return monitorInfo; + + } */ + + + /* + * The following fields & methods are for `collectionmetrics.jsp' + */ + public synchronized SortedMap> + getAllItemCollectionMetrics(String clusterName, long beg, long end) throws SQLException { + + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT item_name, detection_timestamp, time_consumed " + + "FROM " + collectionMetricTableName + " " + + "WHERE cluster_name=? " + + "AND detection_timestamp>? " + + "AND detection_timestamp<=?"); + pstat.setString(1, clusterName); + pstat.setString(2, dateFormat.format(new Date(beg))); + pstat.setString(3, dateFormat.format(new Date(end))); + ResultSet res = pstat.executeQuery(); + + SortedMap> metrics = + new TreeMap>(); + + while (res.next()) { + String itemName = res.getString("item_name"); + Timestamp detectionTimestamp = res.getTimestamp("detection_timestamp"); + Long timeConsumed = res.getLong("time_consumed"); + CollectionMetricRecord record = + new CollectionMetricRecord( + clusterName, + itemName, + detectionTimestamp.getTime(), + timeConsumed); + List list = metrics.get(itemName); + if (list == null) { + list = new ArrayList(); + metrics.put(itemName, list); + } + list.add(record); + } + + pstat.close(); + dbConn.close(); + + return metrics; + } + + public synchronized List getCollectionMetrics( + String clusterName, String itemName, long beg, long end) throws SQLException { + + Connection dbConn = createDBConnection(); + PreparedStatement pstat = dbConn.prepareStatement( + "SELECT detection_timestamp, time_consumed " + + "FROM " + collectionMetricTableName + " " + + "WHERE cluster_name=? " + + "AND item_name=? " + + "AND detection_timestamp>? " + + "AND detection_timestamp<=?"); + pstat.setString(1, clusterName); + pstat.setString(2, itemName); + pstat.setString(3, dateFormat.format(new Date(beg))); + pstat.setString(4, dateFormat.format(new Date(end))); + ResultSet res = pstat.executeQuery(); + + List records = + new ArrayList(); + + while (res.next()) { + Timestamp detectionTimestamp = res.getTimestamp("detection_timestamp"); + Long timeConsumed = res.getLong("time_consumed"); + CollectionMetricRecord record = + new CollectionMetricRecord( + clusterName, + itemName, + detectionTimestamp.getTime(), + timeConsumed); + records.add(record); + } + + pstat.close(); + dbConn.close(); + + return records; + } + + /** + * For daily inspection function by weizhonghui@baidu.com + */ + public synchronized Map checkInspectionResult( + String clusterName, + Map inspectEntry, String begStr, String endStr) throws SQLException { + + Map checkResult = new HashMap(); + + Connection dbConn = createDBConnection(); + + + Iterator> iter = inspectEntry.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String strategyName = entry.getKey(); + + + PreparedStatement pstat = null; + + pstat = dbConn.prepareStatement( + "SELECT detection_timestamp, info_level, info_content " + + "FROM " + inspectionTable + " " + + "WHERE cluster_name=? " + + "AND strategy_name=? " + + "AND detection_timestamp>? " + + "AND detection_timestamp inspectEntry = new LinkedHashMap(); + inspectEntry.put("HDFS_TotalStorageURate_MON", "HdfsUse"); + inspectEntry.put("HDFS_Http4Inode_Collector", "Inodes"); + inspectEntry.put("HDFS_Http4Blocks_Collector", "Blocks"); + inspectEntry.put("HDFS_MissingBlocks_MON", "MissingBlk"); + inspectEntry.put("HDFS_RecentlyRsyncedMetadataFileStatus_MON", "RsyncStat"); + inspectEntry.put("HDFS_LiveDatanodeXceiverUsage_MON", "Xceiver"); + inspectEntry.put("HDFS_LiveDatanodeStorageUsageRate_MON", "DnUse"); + inspectEntry.put("HDFS_LiveDatanodeNonDFSStorageUsage_MON", "NonDfsUse"); + inspectEntry.put("HDFS_LiveDatanodeHeartBeatInterval_MON", "DnHeart"); + inspectEntry.put("HDFS_MultiDatanodeVolumeCorruption_MON", "VolCrpt"); + inspectEntry.put("Datanode_LiveToDeadRepeatedly_Mon", "ReDeadLive"); + inspectEntry.put("Namenode_CheckPointStatus_Mon", "CheckPoint Status"); + } + +} diff --git a/src/java/hma/web/HmaMonitorManager.java b/src/java/hma/web/HmaMonitorManager.java new file mode 100644 index 0000000..c51ffd6 --- /dev/null +++ b/src/java/hma/web/HmaMonitorManager.java @@ -0,0 +1,68 @@ +package hma.web; + +import hma.monitor.HMAMonitorProtocol; +import hma.monitor.MonitorStatus; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; +import java.text.SimpleDateFormat; +import java.util.Iterator; +import java.util.Map; + +/** + * Created with IntelliJ IDEA. + * User: jiangtao02 + * Date: 13-4-28 + * Time: 下午4:11 + * To change this template use File | Settings | File Templates. + */ +public class HmaMonitorManager extends HttpServlet { + + public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException { + + String cmd = req.getParameter("cmd"); + String CLUSTER = req.getParameter("cluster"); + PrintWriter out = res.getWriter(); + SimpleDateFormat format = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss" ); + + if(cmd.equals("get")){ + JSONArray array = new JSONArray(); + Map map = HttpServer.getRpcClient(); + Iterator> iter = map.entrySet().iterator(); + while(iter.hasNext()){ + Map.Entry entry = iter.next(); + String cluster = entry.getKey(); + HMAMonitorProtocol p = entry.getValue(); + MonitorStatus status = p.getAlarmStatus(cluster); + + JSONObject j = new JSONObject(); + try { + j.put("cluster", cluster); + j.put("begin", format.format(status.getBegin())); + j.put("end", format.format(status.getEnd())); + j.put("remain", (status.getEnd() - System.currentTimeMillis())/1000/60); + j.put("isAlarm", status.getAlarm()); + array.put(j); + } catch (JSONException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + + } + out.write(array.toString()); + } else if(cmd.equals("stop")) { + Map map = HttpServer.getRpcClient(); + HMAMonitorProtocol p = map.get(CLUSTER); + p.suspendAlarm(CLUSTER); + } else if(cmd.equals("recover")) { + Map map = HttpServer.getRpcClient(); + HMAMonitorProtocol p = map.get(CLUSTER); + p.recoverAlarm(CLUSTER); + } + } +} diff --git a/src/java/hma/web/HttpServer.java b/src/java/hma/web/HttpServer.java new file mode 100644 index 0000000..3080695 --- /dev/null +++ b/src/java/hma/web/HttpServer.java @@ -0,0 +1,322 @@ +/** + * + */ +package hma.web; + +import hma.conf.Configuration; +import hma.monitor.HMAMonitorProtocol; +import hma.util.HmaStatusJsonIf; +import hma.util.LOG; +import hma.web.oob.OOBMonitorImportDBDumper; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.net.NetUtils; +import org.mortbay.jetty.Connector; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.nio.SelectChannelConnector; +import org.mortbay.jetty.servlet.Context; +import org.mortbay.jetty.servlet.ServletHolder; +import org.mortbay.jetty.webapp.WebAppContext; +import org.mortbay.thread.QueuedThreadPool; +import org.mortbay.util.MultiException; + +import javax.servlet.http.HttpServlet; +import java.io.*; +import java.net.BindException; +import java.net.InetSocketAddress; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + + +/** + * @author guoyezhi + */ +public class HttpServer { + + protected final Server webServer; + + protected final Connector listener; + + protected final boolean findPort; + + protected final Map defaultContexts = + new HashMap(); + + private static final int MAX_RETRIES = 10; + + private WebAppContext webapp; + + private static Map monitorControl = new HashMap(); + + + /** + * @throws IOException + */ + public HttpServer(String name, String bindAddress, int port, + boolean findPort) throws IOException { + this(name, bindAddress, port, findPort, + new Configuration(true, "hma")); + } + + public HttpServer(String name, String bindAddress, int port, + boolean findPort, Configuration conf) throws IOException { + + this.webServer = new Server(); + this.findPort = findPort; + + /** add Connector */ + this.listener = createBaseListener(conf); + if (conf == null) { + this.listener.setHost(name); + this.listener.setPort(port); + } else { + System.out.println(conf.get("web.http.address.name")); + this.listener.setHost( + conf.get("web.http.address.name", "hma.dmop.baidu.com")); + System.out.println(conf.get("web.http.address.port")); + this.listener.setPort(conf.getInt("web.http.address.port", 80)); + } + this.webServer.addConnector(listener); + + /** set thread pool */ + this.webServer.setThreadPool(new QueuedThreadPool()); + + /** set handler */ + webapp = new WebAppContext(); + webapp.setResourceBase(conf.get("web.http.app.resource.base")); + webapp.setContextPath("/"); + webServer.setHandler(webapp); + + /* + final String appDir = getWebAppsPath(); + webAppContext = new WebAppContext(); + webAppContext.setContextPath("/hma"); + //webAppContext.setWar(appDir + "/" + name); + webAppContext.setWar("D:\\eclipse\\guoyezhi\\workspace\\HMA\\webapps\\hma"); + webAppContext.setLogUrlOnStart(false); + webServer.addHandler(new ResourceHandler()); + webServer.setHandler(webAppContext); + System.out.println(webAppContext.getResourceBase()); + + + ContextHandler context = new ContextHandler(); + context.setContextPath("/hma"); + context.setResourceBase("D:\\eclipse\\guoyezhi\\workspace\\HMA\\webapps"); + context.setClassLoader(Thread.currentThread().getContextClassLoader()); + context.setHandler(new ResourceHandler()); + webServer.setHandler(context); + */ + + /* + ResourceHandler publicDocs = new ResourceHandler(); + publicDocs.setResourceBase("D:\\eclipse\\guoyezhi\\workspace\\HMA\\webapps\\hma"); + publicDocs.setWelcomeFiles(new String[]{"index.htm"}); + webServer.setHandler(publicDocs); + */ + + //initRpcClient(); + } + + public static Map getRpcClient() { + initRpcClient(); + return monitorControl; + } + + protected Connector createBaseListener(Configuration conf) + throws IOException { + SelectChannelConnector ret = new SelectChannelConnector(); + ret.setLowResourceMaxIdleTime(10000); + ret.setAcceptQueueSize(128); + ret.setResolveNames(false); + ret.setUseDirectBuffers(false); + return ret; + } + + /** + * Get the pathname to the webapps files. + * + * @return the pathname as a URL + * @throws IOException if 'webapps' directory cannot be found on CLASSPATH. + */ + public String getWebAppsPath() throws IOException { + URL url = getClass().getClassLoader().getResource("webapps"); + if (url == null) + throw new IOException("webapps not found in CLASSPATH"); + return url.toString(); + } + + /** + * Start the server. Does not wait for the server to start. + */ + public void start() throws IOException { + try { + int port = 0; + int oriPort = listener.getPort(); // The original requested port + while (true) { + try { + port = webServer.getConnectors()[0].getLocalPort(); + LOG.info("Port returned by webServer.getConnectors()[0]." + + "getLocalPort() before open() is " + port + + ". Opening the listener on " + oriPort); + listener.open(); + port = listener.getLocalPort(); + LOG.info("listener.getLocalPort() returned " + + listener.getLocalPort() + + " webServer.getConnectors()[0].getLocalPort() returned " + + webServer.getConnectors()[0].getLocalPort()); + // Workaround to handle the problem reported in HADOOP-4744 + if (port < 0) { + Thread.sleep(100); + int numRetries = 1; + while (port < 0) { + LOG.warn("listener.getLocalPort returned " + port); + if (numRetries++ > MAX_RETRIES) { + throw new Exception( + " listener.getLocalPort is returning " + + "less than 0 even after " + + numRetries + " resets"); + } + for (int i = 0; i < 2; i++) { + LOG.info("Retrying listener.getLocalPort()"); + port = listener.getLocalPort(); + if (port > 0) { + break; + } + Thread.sleep(200); + } + if (port > 0) { + break; + } + LOG.info("Bouncing the listener"); + listener.close(); + Thread.sleep(1000); + listener.setPort(oriPort == 0 ? 0 : (oriPort += 1)); + listener.open(); + Thread.sleep(100); + port = listener.getLocalPort(); + } + } // Workaround end + LOG.info("Jetty bound to port " + port); + webServer.start(); + + break; + } catch (IOException ex) { + // if this is a bind exception, + // then try the next port number. + if (ex instanceof BindException) { + if (!findPort) { + throw (BindException) ex; + } + } else { + LOG.info("HttpServer.start() threw a non Bind IOException"); + throw ex; + } + } catch (MultiException ex) { + LOG.info("HttpServer.start() threw a MultiException"); + throw ex; + } + listener.setPort((oriPort += 1)); + } + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException("Problem starting http server", e); + } + } + + /** + * stop the server + */ + public void stop() throws Exception { + listener.close(); + webServer.stop(); + } + + public void join() throws InterruptedException { + webServer.join(); + } + + public void addServlet(String name, String pathSpec, + Class servletClass) { + ServletHolder holder = new ServletHolder(servletClass); + if (name != null) { + holder.setName(name); + } + webapp.addServlet(holder, pathSpec); + } + + private static void initRpcClient() { + File parent = new File(System.getProperty("hma.home.dir") + "/conf/"); + LOG.info("Begin to initRpcClient : " + System.getProperty("hma.home.dir") + "/conf/"); + for (File current : parent.listFiles()) { + if (current.isDirectory()) { + for (File child : current.listFiles()) { + if (child.isFile() && child.getName().equals("hma-rpc.prop")) { + LOG.info("Found hma-rpc.prop at " + child.getAbsoluteFile()); + try { + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(child))); + String data; + String cluster; + String host; + int port; + while ((data = br.readLine()) != null) { + if (data.split(":").length == 3) { + cluster = data.split(":")[0]; + host = data.split(":")[1]; + port = Integer.parseInt(data.split(":")[2]); + + InetSocketAddress addr; + addr = NetUtils.createSocketAddr(host, port); + HMAMonitorProtocol p = (HMAMonitorProtocol) RPC.waitForProxy(HMAMonitorProtocol.class, 0, addr, new org.apache.hadoop.conf.Configuration()); + if (monitorControl.containsKey(cluster)) { + RPC.stopProxy(monitorControl.get(cluster)); + } + monitorControl.put(cluster, p); + LOG.info("hma-rpc.prop loaded for " + cluster); + } + } + } catch (FileNotFoundException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } catch (IOException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + } + } + } + } + } + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + + System.out.println("###Servlet has been added for the first time "); + + int tmpInfoPort = 8090; + String infoHost = "127.0.0.1"; + Configuration hmaConf = new Configuration(true, "hma"); + + HttpServer infoServer = new HttpServer( + "hma-master-host", + infoHost, + tmpInfoPort, + tmpInfoPort == 0, + hmaConf); + + infoServer.start(); + infoServer.addServlet("hmastatus", "/hmastatus", HmaStatusJsonIf.class); + infoServer.addServlet("manager", "/manage", HmaMonitorManager.class); + infoServer.addServlet("inspection", "/inspection", HmaInspection.class); + System.out.println("###Servlet has been added "); + System.out.print("Hello"); + + + new Thread(new OOBMonitorImportDBDumper(hmaConf)).start(); + + } + + +} + diff --git a/src/java/hma/web/oob/OOBMonitorImportDBDumper.java b/src/java/hma/web/oob/OOBMonitorImportDBDumper.java new file mode 100644 index 0000000..6891ad9 --- /dev/null +++ b/src/java/hma/web/oob/OOBMonitorImportDBDumper.java @@ -0,0 +1,112 @@ +/** + * + */ +package hma.web.oob; + +import hma.conf.Configuration; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.TimeUnit; + +/** + * @author guoyezhi + * + */ +public class OOBMonitorImportDBDumper implements Runnable { + + private Configuration hmaConf = null; + + private static SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd" + " " + "HH:mm:ss"); + + private String dbServer = null; + private String dbName = null; + private String tableName = null; + private String userName = null; + private String password = null; + + + public OOBMonitorImportDBDumper(Configuration hmaConf) { + this.hmaConf = hmaConf; + } + + + private Connection initConnection() throws SQLException { + + dbServer = hmaConf.get( + "monitor.data.db.server", "yf-2950-hma-db.yf01.baidu.com"); + dbName = hmaConf.get( + "monitor.data.db.name", "hma"); + tableName = hmaConf.get( + "monitor.extension.oob.monitor.imports.table", + "monitor_ext_oobmonimports"); + userName = hmaConf.get( + "monitor.db.user.name", "root"); + password = hmaConf.get( + "monitor.db.user.pass", "hmainit"); + + return DriverManager.getConnection( + "jdbc:mysql://" + dbServer + "/" + dbName, + userName, + password); + } + + @Override + public void run() { + + Connection dbConn = null; + try { + dbConn = this.initConnection(); + } catch (SQLException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + while (true) { + OOBMonitorImportRecord record = + OOBMonitorImportPool.pollRecord(); + if (record == null) { + try { + TimeUnit.SECONDS.sleep(60); + } catch (InterruptedException e) { + e.printStackTrace(); + } + continue; + } + PreparedStatement pstat = null; + try { + String detectionTimestamp = + dateFormat.format(new Date(record.getDetectionTimestamp())); + pstat = dbConn.prepareStatement( + "INSERT INTO " + tableName + " VALUES ( ?, ?, ?, ?, ? )"); + pstat.setString(1, record.getClusterName()); + pstat.setString(2, detectionTimestamp); + pstat.setString(3, record.getOOBMonitorName()); + if (record.getOOBMonitorResult().equals("normal")) { + pstat.setInt(4, 0); // 0: normal + } else { + pstat.setInt(4, 1); // 1: abnormal + } + pstat.setString(5, record.getOOBMonitorLog()); + pstat.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } finally { + if (pstat != null) { + try { + pstat.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + } + + } + +} diff --git a/src/java/hma/web/oob/OOBMonitorImportPool.java b/src/java/hma/web/oob/OOBMonitorImportPool.java new file mode 100644 index 0000000..1d722fc --- /dev/null +++ b/src/java/hma/web/oob/OOBMonitorImportPool.java @@ -0,0 +1,26 @@ +/** + * + */ +package hma.web.oob; + +import java.util.LinkedList; +import java.util.Queue; + +/** + * @author guoyezhi + * + */ +public class OOBMonitorImportPool { + + private static Queue pool = + new LinkedList(); + + public synchronized static void offerRecord(OOBMonitorImportRecord record) { + pool.offer(record); + } + + public synchronized static OOBMonitorImportRecord pollRecord() { + return pool.poll(); + } + +} diff --git a/src/java/hma/web/oob/OOBMonitorImportRecord.java b/src/java/hma/web/oob/OOBMonitorImportRecord.java new file mode 100644 index 0000000..f115961 --- /dev/null +++ b/src/java/hma/web/oob/OOBMonitorImportRecord.java @@ -0,0 +1,55 @@ +/** + * + */ +package hma.web.oob; + +/** + * @author guoyezhi + * + */ +public class OOBMonitorImportRecord { + + private String clusterName = null; + + private long detectionTimestamp = 0; + + private String OOBMonitorName = null; + + private String OOBMonitorResult = null; + + private String OOBMonitorLog = null; + + public OOBMonitorImportRecord( + String clusterName, + long detectionTimestamp, + String OOBMonitorName, + String OOBMonitorResult, + String OOBMonitorLog) { + this.clusterName = clusterName; + this.detectionTimestamp = detectionTimestamp; + this.OOBMonitorName = OOBMonitorName; + this.OOBMonitorResult = OOBMonitorResult; + this.OOBMonitorLog = OOBMonitorLog; + } + + public String getClusterName() { + return clusterName; + } + + public long getDetectionTimestamp() { + return detectionTimestamp; + } + + public String getOOBMonitorName() { + return OOBMonitorName; + } + + public String getOOBMonitorResult() { + return OOBMonitorResult; + } + + public String getOOBMonitorLog() { + return OOBMonitorLog; + } + +} diff --git a/src/test b/src/test new file mode 100644 index 0000000..01e79c3 --- /dev/null +++ b/src/test @@ -0,0 +1,3 @@ +1 +2 +3