CUDA PHP Integration

Just for fun an integration of Nvidia CUDA capable graphic card into PHP.

Well, mainly it is a CUBLAS(Linear Algebra) Integration patch.
Of course, you need large matrices to benefit from the extra call overhead.
At least the graphic card does not allow concurrent execution, so a webserver integration is questionable. But newer versions of the NVidia CUDA driver (afair) the Fermi architecture can execute concurrent GPU-kernels.

Source: diff patch

diff -urN php-5.3.3/ext/standard//array.c php-5.3.3_new/ext/standard//array.c
--- php-5.3.3/ext/standard//array.c 2010-06-11 10:53:31.000000000 +0200
+++ php-5.3.3_new/ext/standard//array.c 2010-10-12 18:34:38.000000000 +0200
@@ -50,6 +50,10 @@
 #include "ext/spl/spl_array.h"
 #endif
 
+// pkirsch:
+#include "ExcelCUDA_wrapper.h"
+
+
 /* {{{ defines */
 #define EXTR_OVERWRITE   0
 #define EXTR_SKIP    1
@@ -4524,3 +4528,68 @@
  * vim600: noet sw=4 ts=4 fdm=marker
  * vim<600: noet sw=4 ts=4
  */
+
+/* pkirsch CUDA integration:
+ * build:
+ * /usr/local/cuda/bin/nvcc  -gencode=arch=compute_10,code=\"sm_10,compute_10\"  -gencode=arch=compute_20,code=\"sm_20,compute_20\"  -m64 --compiler-options -fno-strict-aliasing  -I. -I/usr/local/cuda/include -I../../common/inc -I../../../shared//inc -DUNIX -O2   -o ExcelCUDA_wrapper.cu.o -c ExcelCUDA_wrapper.cu
+ * - globale Makefile patchen:
+ *  - PHP_GLOBAL_OBJS += ext/standard/ExcelCUDA_wrapper.cu.o
+ *  - EXTRA_LIBS += -lcudart -lcutil_x86_64  -lcublas -L/usr/local/cuda/lib64
+ * 
+ *  */
+
+/* orig: cublasSdot
+ scaling vectors */
+PHP_FUNCTION(cu_scal)
+{
+ zval *input,
+   **entry,
+   entry_n;
+ int error = 0;
+ HashPosition pos;
+ long  scale;
+ 
+ if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "al", &input, &scale) == FAILURE) {
+  return;
+ }
+
+ int array_len = zend_hash_num_elements(Z_ARRVAL_P(input));
+ float *floats_input = malloc(sizeof(float) * array_len);
+ int index = 0;
+ 
+ for (zend_hash_internal_pointer_reset_ex(Z_ARRVAL_P(input), &pos);
+  zend_hash_get_current_data_ex(Z_ARRVAL_P(input), (void **)&entry, &pos) == SUCCESS;
+  zend_hash_move_forward_ex(Z_ARRVAL_P(input), &pos)
+ ) {
+  if (Z_TYPE_PP(entry) == IS_ARRAY || Z_TYPE_PP(entry) == IS_OBJECT) {
+   continue;
+  }
+  entry_n = **entry;
+  zval_copy_ctor(&entry_n);
+  convert_scalar_to_number(&entry_n TSRMLS_CC);  
+  floats_input[index++] = (float)Z_LVAL(entry_n);
+ }
+ 
+ /* CUDA */
+ long time1, time2;
+ time1 = clock();
+ ExcelCUDA_CUBLAS_scal(index,
+       (float)scale,  
+       floats_input,  
+       &error);
+ time2 = clock();
+ printf("cudadiff Timediff: %d\n", time2-time1);
+ time1 = clock();
+ 
+ /* returns an array */
+ array_init_size(return_value, array_len);
+ int index_prev = index;
+    while (index--) {
+     zval *ret;
+     MAKE_STD_ZVAL(ret); /* default zend value allocating */
+     convert_to_double(ret); /* must do: set to type */
+     Z_DVAL_P(ret) = (double)floats_input[index]; /* set result */
+     zval_add_ref(&ret);
+     zend_hash_next_index_insert(Z_ARRVAL_P(return_value), &ret, sizeof(zval *), NULL);
+    }    
+ }
\ Kein Zeilenumbruch am Dateiende.
diff -urN php-5.3.3/ext/standard//basic_functions.c php-5.3.3_new/ext/standard//basic_functions.c
--- php-5.3.3/ext/standard//basic_functions.c 2010-05-13 04:13:30.000000000 +0200
+++ php-5.3.3_new/ext/standard//basic_functions.c 2010-10-12 09:15:21.000000000 +0200
@@ -609,6 +609,13 @@
  ZEND_ARG_INFO(0, keys)   /* ARRAY_INFO(0, keys, 0) */
  ZEND_ARG_INFO(0, values) /* ARRAY_INFO(0, values, 0) */
 ZEND_END_ARG_INFO()
+
+/* cuda */
+ZEND_BEGIN_ARG_INFO_EX(arginfo_cu_scal, 0, 0, 1)
+ ZEND_ARG_INFO(0, input) 
+ ZEND_ARG_INFO(0, scale)
+ZEND_END_ARG_INFO()
+
 /* }}} */
 /* {{{ basic_functions.c */
 ZEND_BEGIN_ARG_INFO(arginfo_get_magic_quotes_gpc, 0)
@@ -3326,7 +3333,9 @@
  PHP_FE(array_chunk,              arginfo_array_chunk)
  PHP_FE(array_combine,             arginfo_array_combine)
  PHP_FE(array_key_exists,            arginfo_array_key_exists)
-
+ /* cuda */
+ PHP_FE(cu_scal,               arginfo_array_chunk)
+ 
  /* aliases from array.c */
  PHP_FALIAS(pos,     current,        arginfo_current)
  PHP_FALIAS(sizeof,    count,         arginfo_count)
diff -urN php-5.3.3/ext/standard//ExcelCUDA_wrapper.cu php-5.3.3_new/ext/standard//ExcelCUDA_wrapper.cu
--- php-5.3.3/ext/standard//ExcelCUDA_wrapper.cu 1970-01-01 01:00:00.000000000 +0100
+++ php-5.3.3_new/ext/standard//ExcelCUDA_wrapper.cu 2010-10-12 10:04:51.000000000 +0200
@@ -0,0 +1,345 @@
+/*
+ * Copyright 1993-2010 NVIDIA Corporation.  All rights reserved.
+ *
+ * Please refer to the NVIDIA end user license agreement (EULA) associated
+ * with this source code for terms and conditions that govern your use of
+ * this software. Any use, reproduction, disclosure, or distribution of
+ * this software and related documentation outside the terms of the EULA
+ * is strictly prohibited.
+ *
+ */
+
+/*
+* Cublas additions: Patrick Kirsch 
+*/
+#include 
+#include 
+
+#include "ExcelCUDA_wrapper.h"
+
+/* #include "reduction.h" */
+#include "cublas.h"
+
+inline int success(cudaError_t result)
+{
+    return result == cudaSuccess;
+}
+
+/* CUBLAS */
+/*   orig: cublasSdot
+ dot product of two vectors
+*/
+void ExcelCUDA_CUBLAS_sdot(int dimension,
+       float *result,  
+       float* h_A, 
+       float* h_B, 
+       int *error) 
+{    
+    cublasStatus status;
+    float* d_A = 0;
+    float* d_B = 0;
+
+    status = cublasInit();
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! CUBLAS initialization error\n");
+        *error = 113;
+    }
+
+    /* Allocate device memory for the matrices */
+    status = cublasAlloc(dimension, sizeof(d_A[0]), (void**)&d_A);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device memory allocation error (A)\n");
+        *error = 114;
+    }
+    status = cublasAlloc(dimension, sizeof(d_B[0]), (void**)&d_B);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device memory allocation error (B)\n");
+        *error = 115;
+    }
+
+    /* Initialize the device matrices with the host matrices */
+    status = cublasSetVector(dimension, sizeof(h_A[0]), h_A, 1, d_A, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (write A)\n");
+        *error = 117;
+    }
+    
+    status = cublasSetVector(dimension, sizeof(h_B[0]), h_B, 1, d_B, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (write B)\n");
+        *error = 118;
+    }
+
+    /* Clear last error */
+    cublasGetError();
+
+    /* Performs operation using cublas */
+    *result = cublasSdot(dimension, d_A, 1, d_B, 1);
+    
+    status = cublasGetError();
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! kernel execution error.\n");
+        *error = -119;
+    }
+    
+    /* Read the result back */
+    //status = cublasGetVector(dimension, sizeof(float), d_B, 1, h_B, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (read B)\n");
+        *error = 121;
+    }
+
+ 
+    /* Memory clean up */
+    free(h_A);
+    free(h_B);
+}
+/*   orig: cublasSdot
+ scaling vectors
+*/
+void ExcelCUDA_CUBLAS_scal(int dimension,
+       float alpha,  
+       float* h_A,  
+       int *error) 
+{    
+    cublasStatus status;
+    float* d_A = 0;
+
+    status = cublasInit();
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! CUBLAS initialization error\n");
+        *error = 113;
+    }
+
+    /* Allocate device memory for the matrices */
+    status = cublasAlloc(dimension, sizeof(d_A[0]), (void**)&d_A);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device memory allocation error (A)\n");
+        *error = 114;
+    }
+
+    /* Initialize the device matrices with the host matrices */
+    status = cublasSetVector(dimension, sizeof(h_A[0]), h_A, 1, d_A, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (write A)\n");
+        *error = 117;
+    }
+    
+    /* Clear last error */
+    cublasGetError();
+
+    /* Performs operation using cublas */
+    cublasSscal(dimension, alpha, d_A, 1);
+    
+    status = cublasGetError();
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! kernel execution error.\n");
+        *error = -119;
+    }
+    
+    /* Read the result back */
+    status = cublasGetVector(dimension, sizeof(float), d_A, 1, h_A, 1);
+     
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (read A)\n");
+        *error = 121;
+    }
+}
+
+/*   orig: cublasSaxpy
+ Y = alpha * X + Y
+*/
+void ExcelCUDA_CUBLAS_saxpy(int dimension,
+       float alpha,  
+       float* h_A, 
+       float* h_B, 
+       int *error) 
+{    
+    cublasStatus status;
+    float* d_A = 0;
+    float* d_B = 0;
+
+    status = cublasInit();
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! CUBLAS initialization error\n");
+        *error = 113;
+    }
+
+    /* Allocate device memory for the matrices */
+    status = cublasAlloc(dimension, sizeof(d_A[0]), (void**)&d_A);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device memory allocation error (A)\n");
+        *error = 114;
+    }
+    status = cublasAlloc(dimension, sizeof(d_B[0]), (void**)&d_B);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device memory allocation error (B)\n");
+        *error = 115;
+    }
+
+    /* Initialize the device matrices with the host matrices */
+    status = cublasSetVector(dimension, sizeof(h_A[0]), h_A, 1, d_A, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (write A)\n");
+        *error = 117;
+    }
+    
+    status = cublasSetVector(dimension, sizeof(h_B[0]), h_B, 1, d_B, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (write B)\n");
+        *error = 118;
+    }
+
+    /* Clear last error */
+    cublasGetError();
+
+    /* Performs operation using cublas */
+    cublasSaxpy(dimension, alpha, d_A, 1, d_B, 1);
+    
+    status = cublasGetError();
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! kernel execution error.\n");
+        *error = -119;
+    }
+    
+    /* Read the result back */
+    status = cublasGetVector(dimension, sizeof(float), d_B, 1, h_B, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (read B)\n");
+        *error = 121;
+    }
+
+ 
+    /* Memory clean up */
+    free(h_A);
+}
+
+/* C = alpha * op(A) * op(B) + beta * C; op(X) = X or op(X) = X transformed 
+Hint:
+ h_(A,B,C) needs to be (float*)malloc(n2 * sizeof(h_A[0])) !
+*/
+void ExcelCUDA_CUBLAS_sgemm(int n2,
+       float alpha, 
+       float beta, 
+       float* h_A, 
+       float* h_B,
+       float* h_C, 
+       int *error) 
+{    
+    cublasStatus status;
+    float* d_A = 0;
+    float* d_B = 0;
+    float* d_C = 0;
+    int dimension = (int)sqrt((float)n2);
+
+    status = cublasInit();
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! CUBLAS initialization error\n");
+        *error = 113;
+    }
+
+    /* Allocate host memory for the matrices 
+    h_A = (float*)malloc(n2 * sizeof(h_A[0]));
+    if (h_A == 0) {
+        fprintf (stderr, "!!!! host memory allocation error (A)\n");
+    }
+    h_B = (float*)malloc(n2 * sizeof(h_B[0]));
+    if (h_B == 0) {
+        fprintf (stderr, "!!!! host memory allocation error (B)\n");
+    }
+    h_C = (float*)malloc(n2 * sizeof(h_C[0]));
+    if (h_C == 0) {
+        fprintf (stderr, "!!!! host memory allocation error (C)\n");
+    } */
+
+    /* Fill the matrices with test data 
+    for (i = 0; i < n2; i++) {
+        h_A[i] = rand() / (float)RAND_MAX;
+        h_B[i] = rand() / (float)RAND_MAX;
+        h_C[i] = rand() / (float)RAND_MAX;
+    }*/
+
+    /* Allocate device memory for the matrices */
+    status = cublasAlloc(n2, sizeof(d_A[0]), (void**)&d_A);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device memory allocation error (A)\n");
+        *error = 114;
+    }
+    status = cublasAlloc(n2, sizeof(d_B[0]), (void**)&d_B);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device memory allocation error (B)\n");
+        *error = 115;
+    }
+    status = cublasAlloc(n2, sizeof(d_C[0]), (void**)&d_C);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device memory allocation error (C)\n");
+        *error = 116;
+    }
+
+    /* Initialize the device matrices with the host matrices */
+    status = cublasSetVector(n2, sizeof(h_A[0]), h_A, 1, d_A, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (write A)\n");
+        *error = 117;
+    }
+    
+    status = cublasSetVector(n2, sizeof(h_B[0]), h_B, 1, d_B, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (write B)\n");
+        *error = 118;
+    }
+    status = cublasSetVector(n2, sizeof(h_C[0]), h_C, 1, d_C, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (write C)\n");
+        *error = 119;
+    }
+    
+    /* Performs operation using plain C code */
+    // not needed: simple_sgemm(N, alpha, h_A, h_B, beta, h_C);
+    // h_C_ref = h_C;
+
+    /* Clear last error */
+    cublasGetError();
+
+    /* Performs operation using cublas */
+    cublasSgemm('n', 'n', dimension, dimension, dimension, alpha, d_A, dimension, d_B, dimension, beta, d_C, dimension);
+    
+    status = cublasGetError();
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! kernel execution error.\n");
+        *error = -119;
+    }
+    
+    /* Allocate host memory for reading back the result from device memory 
+    result = (float*)malloc(n2 * sizeof(h_C[0]));
+    if (*h_C == 0) {
+        fprintf (stderr, "!!!! host memory allocation error (C)\n");
+        *error = 120;
+    }*/
+
+    /* Read the result back */
+    status = cublasGetVector(n2, sizeof(float), d_C, 1, h_C, 1);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf (stderr, "!!!! device access error (read C)\n");
+        *error = 121;
+    }
+
+    /* Check result against reference 
+    error_norm = 0;
+    ref_norm = 0;
+    for (i = 0; i < n2; ++i) {
+        diff = h_C_ref[i] - h_C[i];
+        error_norm += diff * diff;
+        ref_norm += h_C_ref[i] * h_C_ref[i];
+    }
+    error_norm = (float)sqrt((double)error_norm);
+    ref_norm = (float)sqrt((double)ref_norm);
+    if (fabs(ref_norm) < 1e-7) {
+        fprintf (stderr, "!!!! reference norm is 0\n");
+
+    }*/
+ 
+    /* Memory clean up */
+    free(h_A);
+    free(h_B);
+}
diff -urN php-5.3.3/ext/standard//ExcelCUDA_wrapper.h php-5.3.3_new/ext/standard//ExcelCUDA_wrapper.h
--- php-5.3.3/ext/standard//ExcelCUDA_wrapper.h 1970-01-01 01:00:00.000000000 +0100
+++ php-5.3.3_new/ext/standard//ExcelCUDA_wrapper.h 2010-10-11 18:32:10.000000000 +0200
@@ -0,0 +1,25 @@
+/*
+ * Copyright 1993-2010 NVIDIA Corporation.  All rights reserved.
+ *
+ * Please refer to the NVIDIA end user license agreement (EULA) associated
+ * with this source code for terms and conditions that govern your use of
+ * this software. Any use, reproduction, disclosure, or distribution of
+ * this software and related documentation outside the terms of the EULA
+ * is strictly prohibited.
+ *
+ */
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+ void ExcelCUDA_CUBLAS_sgemm(int n2, float alpha, float beta, float* h_A, float* h_B, float* h_C, int *error);
+ void ExcelCUDA_CUBLAS_saxpy(int dimension,float alpha,float* h_A,float* h_B, int *error);
+ void ExcelCUDA_CUBLAS_sdot(int dimension,float *result, float* h_A, float* h_B, int *error) ;
+ void ExcelCUDA_CUBLAS_scal(int dimension,float alpha, float* h_A, int *error);
+#ifdef __cplusplus
+}
+#endif
diff -urN php-5.3.3/ext/standard//php_array.h php-5.3.3_new/ext/standard//php_array.h
--- php-5.3.3/ext/standard//php_array.h 2010-01-03 10:23:27.000000000 +0100
+++ php-5.3.3_new/ext/standard//php_array.h 2010-10-12 09:34:58.000000000 +0200
@@ -102,6 +102,9 @@
 PHP_FUNCTION(array_chunk);
 PHP_FUNCTION(array_combine);
 
+/* cuda */
+PHP_FUNCTION(cu_scal);
+
 PHPAPI HashTable* php_splice(HashTable *, int, int, zval ***, int, HashTable **);
 PHPAPI int php_array_merge(HashTable *dest, HashTable *src, int recursive TSRMLS_DC);
 PHPAPI int php_array_replace_recursive(HashTable *dest, HashTable *src TSRMLS_DC);

Unoconv Universal Konvertierer

Etwas was ich wirklich nahe legen kann ist unoconv. Das Werkzeug aus dem Open Office Paket kommt mit allen gängigen Formaten zurecht (docx,dox,xls,xlsx,odt etc.) und wandelt es in das gewünschte z.B. PDF Format um. Generell gilt, solange OpenOffice (oder LibreOffice) das Format lesen kann, kommt unoconv damit auch zurecht.
Sehr nützlich.
Bsp:

unoconv -f pdf *.doc

MSSQL Solr Import Workaround

After a long while searching for a workaround for inserting data from MSSQL database into a Solr search server a finally did this java server page (jsp):

<html>
<head><title>Workaround the mssql java database driver </title></head>
<body>
<table>
<%@ page import="java.util.*"
 import="java.sql.*"
 import="javax.sql.*" 
 import="javax.sql.*" 
 import="java.io.*" 
 import="java.util.*" 
%>

<%
try {
Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
BufferedOutputStream outPut = new BufferedOutputStream( new FileOutputStream("transferFile.tmp"));
   Connection connection =
DriverManager.getConnection(
"jdbc:sqlserver://192.168.151.151;instanceName=testmssql", "sa","sa");
   System.out.println("Connection Done");
   connection.setCatalog("Test");
   String sqlCmdString = "SELECT dokumentAspdf FROM Test";
   Statement statement = connection.createStatement();
   ResultSet resultSet = statement.executeQuery(sqlCmdString);
   ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
   int columnCount = resultSetMetaData.getColumnCount();
   int tstDocNum=0;
   while (resultSet.next()) {
    tstDocNum++;
     outPut = new BufferedOutputStream( new FileOutputStream(""+resultSet.getString(0)));
     outPut.write(resultSet.getBytes(0));
     out.println("getColumnType:0:"+ resultSet.getString(0));
     outPut.flush();
     outPut.close();
        String args = "/usr/bin/curl "+
                "http://localhost:8080/solr/import/update/extract?map.content=dokumentpdf&map.stream_name=idrowid&commit=true " +
                " -F " +
                " file=@/usr/share/tomcat6/"+resultSet.getString(2);
        Runtime runtime = Runtime.getRuntime();
       Process process = runtime.exec(args);
       InputStream is = process.getInputStream();
       InputStreamReader isr = new InputStreamReader(is);
       BufferedReader br = new BufferedReader(isr);
       String line;
       System.out.printf("exec: "+ args);
        process.waitFor();
       while ((line = br.readLine()) != null) {
         System.out.println(line);
       }
        File f = new File("/usr/share/tomcat6/"+resultSet.getString(2));
        f.delete();
   }
   resultSet.close();
   connection.close();
   System.out.println("Done");
  } catch (SQLException exception) {
   exception.printStackTrace();
  } catch (Exception exception) {
   exception.printStackTrace();
  }
%>
</body>
</html>

Sure it is ugly. But it is triggerable via REST-request.

As you see:

• it reads out the document from the database
• write to a local file
• insert it into solr via curl doing a POST request
  

But finally this fixed the stacktrace:

Qemu, Java and strange errors

Just a hint for all those, who are puzzled with strange Java errors like:

#
# An unexpected error has been detected by Java Runtime Environment:
#
# Internal Error (synchronizer.cpp:1804), pid=10539, tid=2441395088
# Error: guarantee(obj->mark() == markOopDesc::encode(mid),"invariant")
#
# Java VM: Java HotSpot(TM) Client VM (11.3-b02 mixed mode linux-x86)
# If you would like to submit a bug report, please visit:
# http://java.sun.com/webapps/bugreport/crash.jsp
#

I tried OpenSUSE 11.0 with different java-sun (e.g. java-1_6_0-sun-1.6.0.u13-0.1) packages and also Debian with the java standard package. Both in the virtualized environment (as qemu instance), nothing seems to help.

The only thing was the recompilation from (OpenSUSE's) qemu-0.10.1-0.1.1 to qemu-0.10.5 (latest greatest).

Perl Memory Consumption: CollectD

CollectD: Perl

A really nice tool to graphically show system values (like disk free, running processes etc.) is CollectD (They self say: "collectd is a daemon which collects system performance statistics periodically and provides mechanisms to store the values in a variety of ways").

Measure Perl memory consumption

This is a bit tricky within Collectd, esp. when you want to see certain Perl processes. The next problem is the Collectd Perl plugin interface. This hurts really, when you are interested in a perl interpreter which has a small footprint and more: when you want to measure Perl. The Perl plugin interface of Collectd calls simply a Perl interpreter. That is not always wanted.

Perlmemory

(Yes I know, that form of publishing is not the best)

diff -urN collectd-4.5.2/configure.in collectd-4.5.2_a/configure.in
--- collectd-4.5.2/configure.in 2009-01-02 23:18:58.000000000 +0100
+++ collectd-4.5.2_a/configure.in       2009-02-11 09:28:33.000000000 +0100
@@ -2761,6 +2761,7 @@
 AC_PLUGIN([nut],         [$with_libupsclient], [Network UPS tools statistics])
 AC_PLUGIN([onewire],     [$with_libowcapi],    [OneWire sensor statistics])
 AC_PLUGIN([perl],        [$plugin_perl],       [Embed a Perl interpreter])
+AC_PLUGIN([perlmemory],  [yes],              [Perl memory consumption])
 AC_PLUGIN([ping],        [$with_liboping],     [Network latency statistics])
 AC_PLUGIN([postgresql],  [$with_libpq],        [PostgreSQL database statistics])
 AC_PLUGIN([powerdns],    [yes],                [PowerDNS statistics])
@@ -2929,6 +2930,7 @@
     nut . . . . . . . . . $enable_nut
     onewire . . . . . . . $enable_onewire
     perl  . . . . . . . . $enable_perl
+    perlmemory  . . . . . $enable_perlmemory
     ping  . . . . . . . . $enable_ping
     postgresql  . . . . . $enable_postgresql
     powerdns  . . . . . . $enable_powerdns
diff -urN collectd-4.5.2/contrib/collection.cgi collectd-4.5.2_a/contrib/collection.cgi
--- collectd-4.5.2/contrib/collection.cgi       2009-01-02 22:39:54.000000000 +0100
+++ collectd-4.5.2_a/contrib/collection.cgi     2009-02-11 14:24:10.000000000 +0100
@@ -1706,6 +1706,18 @@
     'GPRINT:max:MAX:%5.1lf%sbyte Max,',
     'GPRINT:avg:LAST:%5.1lf%sbyte Last\l'
     ],
+    perlmemory => ['-b', '1024', '-v', 'Bytes',
+    'DEF:avg={file}:value:AVERAGE',
+    'DEF:min={file}:value:MIN',
+    'DEF:max={file}:value:MAX',
+    "AREA:max#$HalfBlue",
+    "AREA:min#$Canvas",
+    "LINE1:avg#$FullBlue:Memory",
+    'GPRINT:min:MIN:%5.1lf%sbyte Min,',
+    'GPRINT:avg:AVERAGE:%5.1lf%sbyte Avg,',
+    'GPRINT:max:MAX:%5.1lf%sbyte Max,',
+    'GPRINT:avg:LAST:%5.1lf%sbyte Last\l'
+    ],
     old_memory => [
     'DEF:used_avg={file}:used:AVERAGE',
     'DEF:free_avg={file}:free:AVERAGE',
@@ -2626,6 +2638,7 @@
   $MetaGraphDefs->{'if_rx_errors'} = \&meta_graph_if_rx_errors;
   $MetaGraphDefs->{'if_tx_errors'} = \&meta_graph_if_rx_errors;
   $MetaGraphDefs->{'memory'} = \&meta_graph_memory;
+  $MetaGraphDefs->{'perlmemory'} = \&meta_graph_perlmemory;
   $MetaGraphDefs->{'nfs_procedure'} = \&meta_graph_nfs_procedure;
   $MetaGraphDefs->{'ps_state'} = \&meta_graph_ps_state;
   $MetaGraphDefs->{'swap'} = \&meta_graph_swap;
@@ -2913,6 +2926,61 @@
   return (meta_graph_generic_stack ($opts, $sources));
 } # meta_graph_memory
+sub meta_graph_perlmemory
+{
+  confess ("Wrong number of arguments") if (@_ != 5);
+
+  my $host = shift;
+  my $plugin = shift;
+  my $plugin_instance = shift;
+  my $type = shift;
+  my $type_instances = shift;
+
+  my $opts = {};
+  my $sources = [];
+
+  $opts->{'title'} = "$host/$plugin"
+  . (defined ($plugin_instance) ? "-$plugin_instance" : '') . "/$type";
+  $opts->{'number_format'} = '%5.1lf%s';
+
+  $opts->{'rrd_opts'} = ['-b', '1024', '-v', 'Bytes'];
+
+  my @files = ();
+
+  $opts->{'colors'} =
+  {
+    'Perl processes'     => '00e000',
+#    'Sum VSZ'   => '0000ff',
+    'Sum RSS' => 'ffb000'
+  };
+
+  for (@$type_instances)
+  {
+    my $inst = $_;
+    my $file = '';
+    my $title = $opts->{'title'};
+
+    for (@DataDirs)
+    {
+      if (-e "$_/$title-$inst.rrd")
+      {
+       $file = "$_/$title-$inst.rrd";
+       last;
+      }
+    }
+    confess ("No file found for $title") if ($file eq '');
+
+    push (@$sources,
+      {
+       name => $inst,
+       file => $file
+      }
+    );
+  } # for (@$type_instances)
+
+  return (meta_graph_generic_stack ($opts, $sources));
+} # meta_graph_perlmemory
+
 sub meta_graph_if_rx_errors
 {
   confess ("Wrong number of arguments") if (@_ != 5);
diff -urN collectd-4.5.2/src/Makefile.am collectd-4.5.2_a/src/Makefile.am
--- collectd-4.5.2/src/Makefile.am      2009-01-02 22:39:55.000000000 +0100
+++ collectd-4.5.2_a/src/Makefile.am    2009-02-11 09:28:33.000000000 +0100
@@ -779,6 +779,14 @@
 collectd_DEPENDENCIES += xmms.la
 endif

+if BUILD_PLUGIN_PERLMEMORY
+pkglib_LTLIBRARIES += perlmemory.la
+perlmemory_la_SOURCES = perlmemory.c
+perlmemory_la_LDFLAGS = -module -avoid-version
+collectd_LDADD += "-dlopen" perlmemory.la
+collectd_DEPENDENCIES += perlmemory.la
+endif
+

 dist_man_MANS = collectd.1 collectd-nagios.1 collectd.conf.5 \
                collectd-email.5 collectd-exec.5 collectd-perl.5 \
diff -urN collectd-4.5.2/src/perlmemory.c collectd-4.5.2_a/src/perlmemory.c
--- collectd-4.5.2/src/perlmemory.c     1970-01-01 01:00:00.000000000 +0100
+++ collectd-4.5.2_a/src/perlmemory.c   2009-02-11 14:24:49.000000000 +0100
@@ -0,0 +1,105 @@
+/**
+ * collectd - Perl memory consumption
+ * Copyright (C) 2005-2007  Patrick Kirsch
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; only version 2 of the License is applicable.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ * Authors:
+ *     Patrick Kirsch 
+ **/
+
+#include "collectd.h"
+#include "common.h"
+#include "plugin.h"
+
+#include 
+#include 
+#include 
+
+
+#ifdef HAVE_SYS_SYSCTL_H
+# include 
+#endif
+
+int splitter(char *buf) {
+        int i;
+        for(i=0;i<=sizeof(buf);i++){
+                if(buf[i] == ' ')
+                       break;
+        }
+       return i;
+}
+
+static void perl_memory_submit (const char *type_instance, gauge_t value)
+{
+       value_t values[1];
+       value_list_t vl = VALUE_LIST_INIT;
+
+       values[0].gauge = value * 1024;
+
+       vl.values = values;
+       vl.values_len = 1;
+       vl.time = time (NULL);
+       sstrncpy (vl.host, hostname_g, sizeof (vl.host));
+       sstrncpy (vl.plugin, "perlmemory", sizeof (vl.plugin));
+       sstrncpy (vl.type, "perlmemory", sizeof (vl.type));
+       sstrncpy (vl.type_instance, type_instance, sizeof (vl.type_instance));
+
+       plugin_dispatch_values (&vl);
+}
+
+static int perl_memory_read (void)
+{
+       int how_many_perl_process = 0;
+
+        long sum_vsz = 0;
+        long sum_rss = 0;
+        int sep_stelle = 0;
+        int i;
+        FILE *pipe;
+
+        pipe = popen("ps -o vsz,rss,cmd -A | grep -i perl | grep -v grep | tr -s ' ' | sed 's/^ //' | cut  -d ' ' -f 1,2;","r");
+        fflush(pipe); /* Weil gepuffert, per default */
+        char buf[128];
+    while(!feof(pipe) ) {
+        if( fgets( buf, 128, pipe ) != NULL ) {
+                how_many_perl_process++;
+                    /*printf("%s<\n", buf );*/
+                sep_stelle = splitter(buf);
+                char tmp[128];
+                strncpy(tmp,buf,sep_stelle);
+                sum_vsz += atol(tmp);
+                for(i=0;i+                        buf[i]=buf[i+sep_stelle];       /* Impliziere das VSZ > RSS (immer!) */
+
+                strcpy(tmp,buf);
+                sum_rss += atol(tmp);
+                /*printf("Erg: %ld %ld\n",sum_vsz, sum_rss);*/
+        }
+    }
+        pclose(pipe);
+
+       if (how_many_perl_process > 0)
+       {
+               perl_memory_submit ("Perl processes",    how_many_perl_process);
+/*             perl_memory_submit ("Sum VSZ", sum_vsz);*/
+               perl_memory_submit ("Sum RSS", sum_rss);
+       }
+       return 0;
+}
+
+void module_register (void)
+{
+       plugin_register_read ("perlmemory", perl_memory_read);
+} /* void module_register */
diff -urN collectd-4.5.2/src/types.db collectd-4.5.2_a/src/types.db
--- collectd-4.5.2/src/types.db 2009-01-02 22:39:55.000000000 +0100
+++ collectd-4.5.2_a/src/types.db       2009-02-11 09:28:33.000000000 +0100
@@ -55,6 +55,7 @@
 memcached_octets       rx:COUNTER:0:4294967295, tx:COUNTER:0:4294967295
 memcached_ops          value:COUNTER:0:134217728
 memory                 value:GAUGE:0:281474976710656
+perlmemory             value:GAUGE:0:281474976710656
 multimeter             value:GAUGE:U:U
 mysql_commands         value:COUNTER:0:U
 mysql_handler          value:COUNTER:0:U

The interesting line is

pipe = popen("ps -o vsz,rss,cmd -A | grep -i perl | grep -v grep | tr -s ' ' | sed 's/^ //' | cut  -d ' ' -f 1,2;","r");

Here you can add your service to track.(Of course, I do understand, that a fork within a library should be circumvented. Nonetheless is that a working example for easily tracking the memory consumption of a Perl process).

Some hints
I hope it makes some sence to you.

Links

CollectD

Update:
Perlmemory plugin for collectd