diff --git a/progress/32bit/cla.inp b/progress/32bit/cla.inp index 93122c6..648b4ac 100644 --- a/progress/32bit/cla.inp +++ b/progress/32bit/cla.inp @@ -16,7 +16,7 @@ dsaa ${ServiceName_C} HPCC-ODBC32_ConnDS DataSourceIPProperties HPCC-ODBC32_Conn saa ${ServiceName_C} ServiceEnvironmentVariable "PATH=${InstallDir}\ip\bin" -dsaa ${ServiceName_C} HPCC-ODBC32_ConnDS DataSourceIPCustomProperties PROTOCOL=http;WSSQLPORT=8510;WSSQLIP=127.0.0.1;CLUSTER=hthor;DEFAULTQUERYSET=thor;MAXROWBUFFCOUNT=10000; +dsaa ${ServiceName_C} HPCC-ODBC32_ConnDS DataSourceIPCustomProperties PROTOCOL=http;WSSQLPORT=8510;WSSQLIP=127.0.0.1;CLUSTER=hthor;DEFAULTQUERYSET=thor;MAXROWBUFFCOUNT=10000;CACHETIMEOUT=30; e "SaveConfig" SaveConfig diff --git a/progress/32bit/readme.txt b/progress/32bit/readme.txt index 5d0975c..63d9eb4 100644 --- a/progress/32bit/readme.txt +++ b/progress/32bit/readme.txt @@ -30,7 +30,7 @@ Once the plug-in is installed, you must set the IP address of the HPCC WsSQL ser 1) In the MMC console, expand the "Console Root" > "Manager" > "C:\Program Files (x86)\HPCCSystems\HPCC-ODBC Connector\cfg\oadm.ini" > "Services" > "HPCC-ODBC_ConnSvc" > "Data Source Settings" > "HPCC-ODBC_ConnDS" 2) Select "IP Parameters" 3) Double-click the "DataSourceIPCustomProperties" and edit any of the values in the key/value pairs listed to settings that you prefer. - The default settings are : PROTOCOL=http;WSSQLPORT=8510;WSSQLIP=127.0.0.1;CLUSTER=hthor;DEFAULTQUERYSET=thor;MAXROWBUFFCOUNT=10000; + The default settings are : PROTOCOL=http;WSSQLPORT=8510;WSSQLIP=127.0.0.1;CLUSTER=hthor;DEFAULTQUERYSET=thor;MAXROWBUFFCOUNT=10000;CACHETIMEOUT=30; 4) You must change the WSSQLIP value to the IP address of the HPCC ESP Server to which the WsSQL service is bound. 5) Ensure the WSSQLPORT is correct. Check with your HPCC system admin. 6) Select OK, the "File > "Save" and agree to all the prompts diff --git a/progress/64bit/cla.inp b/progress/64bit/cla.inp index 6ffe0d0..a748a01 100644 --- a/progress/64bit/cla.inp +++ b/progress/64bit/cla.inp @@ -16,7 +16,7 @@ dsaa ${ServiceName_C} HPCC-ODBC64_ConnDS DataSourceIPProperties HPCC-ODBC64_Conn saa ${ServiceName_C} ServiceEnvironmentVariable "PATH=${InstallDir}\ip\bin" -dsaa ${ServiceName_C} HPCC-ODBC64_ConnDS DataSourceIPCustomProperties PROTOCOL=http;WSSQLPORT=8510;WSSQLIP=127.0.0.1;CLUSTER=hthor;DEFAULTQUERYSET=thor;MAXROWBUFFCOUNT=10000; +dsaa ${ServiceName_C} HPCC-ODBC64_ConnDS DataSourceIPCustomProperties PROTOCOL=http;WSSQLPORT=8510;WSSQLIP=127.0.0.1;CLUSTER=hthor;DEFAULTQUERYSET=thor;MAXROWBUFFCOUNT=10000;CACHETIMEOUT=30; e "SaveConfig" SaveConfig diff --git a/progress/64bit/readme.txt b/progress/64bit/readme.txt index 154a720..daa4439 100644 --- a/progress/64bit/readme.txt +++ b/progress/64bit/readme.txt @@ -30,7 +30,7 @@ Once the plug-in is installed, you must set the IP address of the HPCC WsSQL ser 1) In the MMC console, expand the "Console Root" > "Manager" > "C:\Program Files\HPCCSystems\HPCC-ODBC Connector\cfg\oadm.ini" > "Services" > "HPCC-ODBC_ConnSvc" > "Data Source Settings" > "HPCC-ODBC_ConnDS" 2) Select "IP Parameters" 3) Double-click the "DataSourceIPCustomProperties" and edit any of the values in the key/value pairs listed to settings that you prefer. - The default settings are : PROTOCOL=http;WSSQLPORT=8510;WSSQLIP=127.0.0.1;CLUSTER=hthor;DEFAULTQUERYSET=thor;MAXROWBUFFCOUNT=10000; + The default settings are : PROTOCOL=http;WSSQLPORT=8510;WSSQLIP=127.0.0.1;CLUSTER=hthor;DEFAULTQUERYSET=thor;MAXROWBUFFCOUNT=10000;CACHETIMEOUT=30; 4) You must change the WSSQLIP value to the IP address of the HPCC ESP Server to which the WsSQL service is bound. 5) Ensure the WSSQLPORT is correct. Check with your HPCC system admin. 6) Select OK, the "File > "Save" and agree to all the prompts diff --git a/src/hpcc_drv.cpp b/src/hpcc_drv.cpp index d58024c..afcf87c 100644 --- a/src/hpcc_drv.cpp +++ b/src/hpcc_drv.cpp @@ -105,7 +105,7 @@ const char * pszWSSQLIP = "WSSQLIP="; const char * pszCLUSTER = "CLUSTER="; const char * pszDEFAULTQUERYSET = "DEFAULTQUERYSET="; const char * pszMAXROWBUFFCOUNT = "MAXROWBUFFCOUNT="; - +const char * pszCACHETIMEOUT = "CACHETIMEOUT="; /************************************************************************ Function: OAIP_init() @@ -360,13 +360,15 @@ int OAIP_connect(DAM_HDBC dam_hdbc, IP_HENV henv, defaultQuerySet.set("roxie"); //Isolate Default QuerySet - aindex_t maxFetchRowCount; p = sl_strstr(sIPCustomProperties, pszMAXROWBUFFCOUNT); - if (p) - maxFetchRowCount = p ? atol(p + strlen(pszMAXROWBUFFCOUNT)) : -1;//-1 if no limit + aindex_t maxFetchRowCount = p ? atol(p + strlen(pszMAXROWBUFFCOUNT)) : -1;//-1 if no limit + + //Isolate Cache Timeout + p = sl_strstr(sIPCustomProperties, pszCACHETIMEOUT); + aindex_t cacheTimeout = p ? atol(p + strlen(pszCACHETIMEOUT)) : 30;//30 minutes if not specified /* initialize the IP data source */ - pConnDA->pHPCCdb = new HPCCdb(hpcc_tm_Handle, protocol.str(), wssqlPort.str(), sUserName, sPassword, wssqlIP.str(), targetCluster.str(), defaultQuerySet.str(), maxFetchRowCount); + pConnDA->pHPCCdb = new HPCCdb(hpcc_tm_Handle, protocol.str(), wssqlPort.str(), sUserName, sPassword, wssqlIP.str(), targetCluster.str(), defaultQuerySet.str(), maxFetchRowCount, cacheTimeout); if (!pConnDA->pHPCCdb->getHPCCDBSystemInfo()) { delete pConnDA->pHPCCdb; diff --git a/src/hpccdb.cpp b/src/hpccdb.cpp index dbb55ca..156ea8f 100644 --- a/src/hpccdb.cpp +++ b/src/hpccdb.cpp @@ -28,11 +28,71 @@ static TM_ModuleCB driver_tm_Hdle;//for logging + +//******************************************************************* +//Shared schema cache singleton, persists even when connections close +//******************************************************************* +class CTable; +class CTableSchemaCache : public CInterface, public IInterface +{ +public: + IMPLEMENT_IINTERFACE; + static CTableSchemaCache * queryInstance(); + virtual ~CTableSchemaCache() {} + + void clearCache() + { + CriticalBlock b(m_crit); + m_tableCache.kill(); + m_metaDataCacheComplete = false; + } + + CTable * getValue(const char * tblName) + { + CriticalBlock b(m_crit); + return m_tableCache.getValue(tblName); + } + + void setValue(const char * tblName, Owned *tblEntry) + { + CriticalBlock b(m_crit); + m_tableCache.setValue(tblName, *tblEntry); + } + + void setMetaDataCacheComplete(bool complete)//called when cache has the entire HPCC metadata, to ensure no more calls to WsSQL + { + CriticalBlock b(m_crit); + m_metaDataCacheComplete = complete; + } + + bool queryMetaDataCacheComplete() + { + CriticalBlock b(m_crit); + return m_metaDataCacheComplete; + } + +protected: + CTableSchemaCache() { m_metaDataCacheComplete = false; } + +private: + //attributes + MapStringToMyClass m_tableCache;//Table Schema Cache + CriticalSection m_crit; + bool m_metaDataCacheComplete; +}; + +CTableSchemaCache *CTableSchemaCache::queryInstance() +{ + static CTableSchemaCache Instance; + return &Instance; +} + + /************************************************************************ Class: HPCCdb *************************************************************************/ -HPCCdb::HPCCdb(TM_ModuleCB tmHandle, const char * _protocol, const char * _wsSqlPort, const char * _user, const char * _pwd, const char * _wssqlIP, const char * _targetCluster, const char * _defaultQuerySet, aindex_t _maxFetchRowCount) +HPCCdb::HPCCdb(TM_ModuleCB tmHandle, const char * _protocol, const char * _wsSqlPort, const char * _user, const char * _pwd, const char * _wssqlIP, const char * _targetCluster, const char * _defaultQuerySet, aindex_t _maxFetchRowCount, aindex_t _cacheTimeout) { tm_trace(driver_tm_Hdle, UL_TM_INFO, "HPCC_Conn:call to HPCCdb::HPCCdb()\n", ()); driver_tm_Hdle = tmHandle; @@ -47,8 +107,9 @@ HPCCdb::HPCCdb(TM_ModuleCB tmHandle, const char * _protocol, const char * _wsSql m_targetCluster.set(_targetCluster); m_defaultQuerySet.set(_defaultQuerySet); m_maxFetchRowCount = _maxFetchRowCount; + m_cacheTimeout = _cacheTimeout; - tm_trace(driver_tm_Hdle, UL_TM_INFO, "HPCC_Conn: protocol='%s', wsSqlPort='%s', user='%s', wssqlIP='%s', targetCluster='%s', defaultQuerySet='%s', maxFetchRowCount='%d'\n", (_protocol,_wsSqlPort,_user,_wssqlIP,_targetCluster,_defaultQuerySet,_maxFetchRowCount)); + tm_trace(driver_tm_Hdle, UL_TM_INFO, "HPCC_Conn: protocol='%s', wsSqlPort='%s', user='%s', wssqlIP='%s', targetCluster='%s', defaultQuerySet='%s', maxFetchRowCount='%d', cacheTimeout=%d\n", (_protocol,_wsSqlPort,_user,_wssqlIP,_targetCluster,_defaultQuerySet,_maxFetchRowCount,_cacheTimeout)); //Create connection object m_clientWs_sql.setown(createwssqlClient()); @@ -272,99 +333,117 @@ bool HPCCdb::getTableSchema(const char * _tableFilter, IArrayOf &_tables { tm_trace(driver_tm_Hdle, UL_TM_INFO, "HPCC_Conn:call to HPCCdb::getTableSchema('%s')\n", (_tableFilter)); + CTableSchemaCache * pCache = CTableSchemaCache::queryInstance(); + + static time_t m_lastCacheClear = 0; + if ((msTick() - m_lastCacheClear) >= (queryCacheTimeout() * 60 * 1000)) + { + pCache->clearCache(); + m_lastCacheClear = msTick(); + } + //First look in the schemaCache if (_tableFilter) { - CTable * cachedEntry = m_tableSchemaCache.getValue(_tableFilter); + CTable * cachedEntry = pCache->getValue(_tableFilter); if (cachedEntry) { tm_trace(driver_tm_Hdle, UL_TM_INFO, "HPCC_Conn:using cached table schema info\n", ()); _tables.append(*(LINK(cachedEntry))); return true; } + else if (pCache->queryMetaDataCacheComplete())//we have everything, and it's not here + return false; } - //Call out to ws_sql to get schema - Owned req; - Owned resp; - - req.setown( createClientGetDBMetaDataRequest()); - req->setIncludeTables(true); - req->setTableFilter(_tableFilter); - req->setIncludeStoredProcedures(false); - CriticalBlock b(crit); - try + if (!pCache->queryMetaDataCacheComplete())// { - tm_trace(driver_tm_Hdle, UL_TM_INFO, "HPCC_Conn:calling ws_sql.GetDBMetaData()...\n", ()); - resp.setown(m_clientWs_sql->GetDBMetaData(req));//calls ws_sql - tm_trace(driver_tm_Hdle, UL_TM_INFO, "HPCC_Conn:complete\n", ()); - } + //Call out to ws_sql to get schema + Owned req; + Owned resp; - catch (IException* e) - { - StringBuffer sb; - e->errorMessage(sb); + req.setown( createClientGetDBMetaDataRequest()); + req->setIncludeTables(true); + req->setTableFilter(_tableFilter); + req->setIncludeStoredProcedures(false); + CriticalBlock b(crit); + try + { + tm_trace(driver_tm_Hdle, UL_TM_INFO, "HPCC_Conn:calling ws_sql.GetDBMetaData()...\n", ()); + resp.setown(m_clientWs_sql->GetDBMetaData(req));//calls ws_sql + tm_trace(driver_tm_Hdle, UL_TM_INFO, "HPCC_Conn:complete\n", ()); + } - hpccErrors.setf("HPCC_Conn:Error calling ws_sql : '%s'\n", (sb.str())); - tm_trace(driver_tm_Hdle, UL_TM_ERRORS, "%s", (sb.str())); - return false; - } + catch (IException* e) + { + StringBuffer sb; + e->errorMessage(sb); - catch (...) - { - hpccErrors.setf("Exception thrown calling ws_sql"); - tm_trace(driver_tm_Hdle, UL_TM_ERRORS, "HPCC_Conn:Error calling ws_sql\n", ()); - return false; - } + hpccErrors.setf("HPCC_Conn:Error calling ws_sql : '%s'\n", (sb.str())); + tm_trace(driver_tm_Hdle, UL_TM_ERRORS, "%s", (sb.str())); + return false; + } - StringBuffer sbErrors; - if (checkForTopLevelErrors(resp->getExceptions(), sbErrors)) - return false; + catch (...) + { + hpccErrors.setf("Exception thrown calling ws_sql"); + tm_trace(driver_tm_Hdle, UL_TM_ERRORS, "HPCC_Conn:Error calling ws_sql\n", ()); + return false; + } - if (!_tableFilter) - m_tableSchemaCache.kill();//empty cache + StringBuffer sbErrors; + if (checkForTopLevelErrors(resp->getExceptions(), sbErrors)) + return false; - //-------------------- - //Tables - //-------------------- - IArrayOf &tables = resp->getTables(); - ForEachItemIn(tableIdx, tables)//process all tables - { - //Add all found tables and their columns to the schemaCache - CHPCCTable &tableItem = (CHPCCTable &)tables.item(tableIdx); - tm_trace(driver_tm_Hdle, UL_TM_F_TRACE, "HPCC_Conn:Found table '%s'\n", (tableItem.getName())); - Owned tblEntry; - tblEntry.setown(new CTable(tableItem.getName(), tableItem.getDescription(), tableItem.getOwner())); + if (!_tableFilter) + { + pCache->clearCache();//empty cache + pCache->setMetaDataCacheComplete(true); + m_lastCacheClear = msTick();//reset cache expiration + } + + //-------------------- + //Tables + //-------------------- + IArrayOf &tables = resp->getTables(); + ForEachItemIn(tableIdx, tables)//process all tables + { + //Add all found tables and their columns to the schemaCache + CHPCCTable &tableItem = (CHPCCTable &)tables.item(tableIdx); + tm_trace(driver_tm_Hdle, UL_TM_F_TRACE, "HPCC_Conn:Found table '%s'\n", (tableItem.getName())); + Owned tblEntry; + tblEntry.setown(new CTable(tableItem.getName(), tableItem.getDescription(), tableItem.getOwner())); #ifdef _WIN32 - OutputDebugString(tableItem.getName());OutputDebugString("\n"); + OutputDebugString(tableItem.getName());OutputDebugString("\n"); #endif - IArrayOf &columns = tableItem.getColumns(); - ForEachItemIn(columnsIdx, columns)//process all columns in given table - { - CHPCCColumn &columnItem = (CHPCCColumn &)columns.item(columnsIdx); - Owned columnEntry; - tm_trace(driver_tm_Hdle, UL_TM_F_TRACE, " HPCC:Found column '%s'(%s)\n", (columnItem.getName(),columnItem.getType())); - columnEntry.setown(new CColumn(columnItem.getName())); + IArrayOf &columns = tableItem.getColumns(); + ForEachItemIn(columnsIdx, columns)//process all columns in given table + { + CHPCCColumn &columnItem = (CHPCCColumn &)columns.item(columnsIdx); + Owned columnEntry; + tm_trace(driver_tm_Hdle, UL_TM_F_TRACE, " HPCC:Found column '%s'(%s)\n", (columnItem.getName(),columnItem.getType())); + columnEntry.setown(new CColumn(columnItem.getName())); #ifdef _WIN32 - StringBuffer sb; - sb.appendf("\t%s (%s)\n",columnItem.getName(), columnItem.getType()); - OutputDebugString(sb.str()); + StringBuffer sb; + sb.appendf("\t%s (%s)\n",columnItem.getName(), columnItem.getType()); + OutputDebugString(sb.str()); #endif - columnEntry->m_hpccType.set(columnItem.getType()); + columnEntry->m_hpccType.set(columnItem.getType()); - //Add to column array - tblEntry->addColumn(LINK(columnEntry)); - } - m_tableSchemaCache.setValue(tableItem.getName(), tblEntry);//save in the schemaCache - if (_tableFilter == NULL) - { - _tables.append(*(LINK(tblEntry))); + //Add to column array + tblEntry->addColumn(LINK(columnEntry)); + } + pCache->setValue(tableItem.getName(), &tblEntry);//save in the schemaCache + if (_tableFilter == NULL) + { + _tables.append(*(LINK(tblEntry))); + } } } if (_tableFilter) { - CTable * cachedEntry = m_tableSchemaCache.getValue(_tableFilter); + CTable * cachedEntry = pCache->getValue(_tableFilter); if (cachedEntry) { _tables.append(*(LINK(cachedEntry))); @@ -455,7 +534,8 @@ bool HPCCdb::executeSQL(const char * sql, const char * targetQuerySet, StringBuf if (targetQuerySet && *targetQuerySet) req->setTargetQuerySet(targetQuerySet); - req->setTargetCluster(m_targetCluster); + if (strnicmp(sql, "call", 4)) + req->setTargetCluster(m_targetCluster);//not allowed on CALL CriticalBlock b(crit); try diff --git a/src/hpccdb.hpp b/src/hpccdb.hpp index b6f3e5d..0753836 100644 --- a/src/hpccdb.hpp +++ b/src/hpccdb.hpp @@ -180,9 +180,9 @@ class HPCCdb StringAttr m_defaultQuerySet;//thor, roxie, thor_roxie, etc StringAttr m_FullHPCCVersion; aindex_t m_maxFetchRowCount;//max num rows to fetch in a single retrieval + aindex_t m_cacheTimeout;//in minutes Owned m_clientWs_sql;//ws_sql client - MapStringToMyClass m_tableSchemaCache; vector m_hpccColPtrs; //Ordered array of pointers to output CColumns that reside in m_schemaCache @@ -203,7 +203,7 @@ class HPCCdb bool checkForErrors(const IMultiException & _exc, IConstECLWorkunit & _wu, StringBuffer & _sbErrors); public: - HPCCdb(TM_ModuleCB tmHandle, const char * _protocol, const char * _wsSqlPort, const char * _user, const char * _pwd, const char * _wssqlIP, const char * _targetCluster, const char * _defaultQuerySet, aindex_t _maxFetchRowCount); + HPCCdb(TM_ModuleCB tmHandle, const char * _protocol, const char * _wsSqlPort, const char * _user, const char * _pwd, const char * _wssqlIP, const char * _targetCluster, const char * _defaultQuerySet, aindex_t _maxFetchRowCount, aindex_t _cacheTimeout); virtual ~HPCCdb(); const char * queryUserName() { return m_userName; } @@ -216,6 +216,7 @@ class HPCCdb const char *queryTargetCluster() { return m_targetCluster; } const char *queryDefaultQuerySet() { return m_defaultQuerySet; } aindex_t queryMaxFetchRowCount() { return m_maxFetchRowCount; } + aindex_t queryCacheTimeout() { return m_cacheTimeout; } const char *queryCurrentQuerySet() { return m_currentQuerySet.get(); } void setCurrentQuerySet(const char * qs) { m_currentQuerySet.set(qs); }