Merge branch 'master' of github.com:lenchv/hive-js

.docker/Makefile

@@ -23,6 +23,11 @@ up-krb:
 	&& docker-compose exec -T kerberos.server kadmin -p admin/admin -w admin -q "addprinc -pw hive [email protected]" \
 	&& cp ./confs/hive-site.xml.$(TYPE) ./confs/hive-site.xml \
 	&& docker-compose up -d hive-server hive-metastore
+up-zoo:
+	rm -f ./confs/hive-site.xml \
+	&& cp ./confs/hive-site.xml.$(TYPE) ./confs/hive-site.xml \
+	&& sed "s/HOSTNAME_VALUE/`hostname -f`/g" docker-compose.zookeeper.dist > docker-compose.yml \
+	&& docker-compose up -d
 down:
 	docker-compose down -v
 log:

.docker/confs/hive-site.xml.tcp.nosasl

@@ -51,6 +51,4 @@
         <name>datanucleus.schema.autoCreateTables</name>
         <value>true</value>
     </property>
-
 </configuration>
-

.docker/confs/hive-site.xml.tcp.nosasl.zoo

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl" ?>
+<configuration>
+    <property>
+        <name>hive.server2.thrift.bind.host</name>
+        <value>0.0.0.0</value>
+    </property>
+    <property>
+        <name>hive.server2.transport.mode</name>
+        <value>binary</value>
+    </property>
+    <property>
+        <name>hive.server2.thrift.http.port</name>
+        <value>10001</value>
+    </property>
+    <property>
+        <name>hive.server2.thrift.port</name>
+        <value>10000</value>
+    </property>
+    <property>
+        <name>hive.server2.thrift.http.path</name>
+        <value>/hive</value>
+    </property>
+    <property>
+        <name>hive.server2.authentication</name>
+        <value>nosasl</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.schema.verification</name>
+        <value>false</value>
+    </property>
+    <property>
+        <name>hive.metastore.warehouse.dir</name>
+        <value>file:///tmp</value>
+        <description></description>
+    </property>
+    <property>
+        <name>fs.default.name</name>
+        <value>file:///tmp</value>
+    </property>
+    <property>
+        <property>
+            <name>javax.jdo.option.ConnectionURL</name>
+            <value>jdbc:derby:;databaseName=metastore_db;create=true</value>
+        </property>
+        <name>javax.jdo.option.ConnectionDriverName</name>
+        <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+    </property>
+    <property>
+        <name>datanucleus.schema.autoCreateTables</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.zookeeper.quorum</name>
+        <value>zookeeper:2181</value>
+    </property>
+
+    <property>
+        <name>hive.zookeeper.namespace</name>
+        <value>hiveserver2</value>
+    </property>
+
+    <property>
+        <name>hive.server2.support.dynamic.service.discovery</name>
+        <value>true</value>
+    </property>
+
+</configuration>

.docker/docker-compose.zookeeper.dist

@@ -0,0 +1,45 @@
+version: "3"
+
+services:
+  hive-server:
+    image: lenchv/hive:2.3.8
+    ports:
+      - "10000:10000"
+      - "10001:10001"
+      - "10002:10002"
+    command: /opt/apache-hive-2.3.6-bin/bin/hiveserver2
+    hostname: HOSTNAME_VALUE
+    volumes:
+      - ./confs/hive-site.xml:/opt/apache-hive-2.3.6-bin/conf/hive-site.xml
+      - ./ssl/:/opt/ssl/
+  hive-metastore:
+    image: lenchv/hive:2.3.8
+    hostname: HOSTNAME_VALUE
+    ports:
+      - "9083:9083"
+    volumes:
+      - ./confs/hive-site.xml:/opt/apache-hive-2.3.6-bin/conf/hive-site.xml
+      - ./ssl/:/opt/ssl/
+    command: /opt/apache-hive-2.3.6-bin/bin/hive --service metastore
+  openldap:
+    image: osixia/openldap:1.2.4
+    hostname: hive.driver
+    environment:
+      LDAP_ORGANISATION: lenchv
+      LDAP_DOMAIN: hive.driver
+      LDAP_ADMIN_PASSWORD: "123456"
+      LDAP_LOG_LEVEL: "256"
+      LDAP_TLS: "false"
+      LDAP_TLS_CRT_FILENAME: "ldap.crt"
+      LDAP_TLS_KEY_FILENAME: "ldap.key"
+      LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem"
+      LDAP_TLS_CA_CRT_FILENAME: "ca.crt"
+      LDAP_TLS_ENFORCE: "false"
+      LDAP_TLS_CIPHER_SUITE: "SECURE256:-VERS-SSL3.0"
+      LDAP_TLS_PROTOCOL_MIN: "3.1"
+      LDAP_TLS_VERIFY_CLIENT: "demand"
+  zookeeper:
+    image: zookeeper:3.7
+    hostname: hive.driver
+    ports:
+      - "2181:2181"

.docker/hive/2.3.8/Dockerfile

@@ -0,0 +1,11 @@
+FROM lenchv/base-hive:2.3.8
+
+ARG USER_UID=1000
+ARG USER_GID=1000
+
+RUN groupadd -f -r hive --gid=$USER_GID && \
+    useradd -r -g hive --uid=$USER_UID -d ${HIVE_HOME} hive && \
+    chown hive:hive -R ${HIVE_HOME}
+
+USER hive
+

.docker/hive/2.3.8/Dockerfile.base

@@ -0,0 +1,22 @@
+FROM openjdk:8u212-jre
+
+WORKDIR /opt
+
+ENV HADOOP_HOME=/opt/hadoop-2.9.2
+ENV HIVE_HOME=/opt/apache-hive-2.3.6-bin
+# Include additional jars
+ENV HADOOP_CLASSPATH=/opt/hadoop-2.9.2/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.199.jar:/opt/hadoop-2.9.2/share/hadoop/tools/lib/hadoop-aws-2.9.2.jar
+
+RUN apt-get update && \
+    curl -L https://www-us.apache.org/dist/hive/hive-2.3.8/apache-hive-2.3.8-bin.tar.gz | tar zxf - && \
+    curl -L https://www-us.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz | tar zxf - && \
+	mv /opt/apache-hive-2.3.8-bin $HIVE_HOME && \
+    mv /opt/hadoop-2.10.1 $HADOOP_HOME
+
+RUN env DEBIAN_FRONTEND=noninteractive apt-get install -y krb5-user
+
+COPY conf ${HIVE_HOME}/conf
+
+WORKDIR $HIVE_HOME
+EXPOSE 9083
+

.docker/hive/2.3.8/conf/hive-log4j2.properties

@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+status = INFO
+name = HiveLog4j2
+packages = org.apache.hadoop.hive.ql.log
+
+# list of properties
+property.hive.log.level = INFO
+property.hive.root.logger = console
+property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}
+property.hive.log.file = hive.log
+property.hive.perflogger.log.level = INFO
+
+# list of all appenders
+appenders = console
+
+# console appender
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+
+# list of all loggers
+loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger
+
+logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
+logger.NIOServerCnxn.level = WARN
+
+logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
+logger.ClientCnxnSocketNIO.level = WARN
+
+logger.DataNucleus.name = DataNucleus
+logger.DataNucleus.level = ERROR
+
+logger.Datastore.name = Datastore
+logger.Datastore.level = ERROR
+
+logger.JPOX.name = JPOX
+logger.JPOX.level = ERROR
+
+logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger
+logger.PerfLogger.level = ${sys:hive.perflogger.log.level}
+
+# root logger
+rootLogger.level = ${sys:hive.log.level}
+rootLogger.appenderRefs = root
+rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}

.docker/hive/2.3.8/conf/hive-site.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+      <property>
+         <name>hive.metastore.schema.verification</name>
+         <value>false</value>
+      </property>
+     <property>
+        <name>hive.metastore.warehouse.dir</name>
+        <value>file:///tmp</value>
+        <description></description>
+     </property>
+     <property>
+        <name>fs.default.name</name>
+        <value>file:///tmp</value>
+     </property>
+     <property>
+     <property>
+        <name>javax.jdo.option.ConnectionURL</name>
+        <value>jdbc:derby:;databaseName=metastore_db;create=true</value>
+     </property>
+        <name>javax.jdo.option.ConnectionDriverName</name>
+        <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+     </property>
+     <property>
+        <name>datanucleus.schema.autoCreateTables</name>
+        <value>true</value>
+     </property>
+</configuration>

CHANGELOG.md

@@ -1,4 +1,11 @@
 # Changelog
+
+[0.1.5](https://github.com/lenchv/hive-driver/releases/tag/v0.1.5) 2021-05-16
+
+- Added method `flush()` to operation class, in order to avoid memory allocation errors when working with massive amount of data
+
+- Added an instruction how to connect via zookeeper using hive-driver
+
 [0.1.4](https://github.com/lenchv/hive-driver/releases/tag/v0.1.4) 2020-07-03
 
 - Fixed retrieving null values from the API

dist/HiveOperation.d.ts

@@ -44,6 +44,11 @@ export default class HiveOperation implements IOperation {
     getSchema(): TableSchema | null;
     getData(): Array<RowSet>;
     getQueryId(): Promise<string>;
+    /**
+     * Resets `this.data` buffer.
+     * Needs to be called when working with massive data.
+     */
+    flush(): void;
     /**
      * Retrieves schema
      * @throws {StatusError}