/*
This file is part of the software framework JStatCom,
a library for building applications for data based analysis.
Copyright (C) 2000-2005 Markus Kraetzig
This library is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation; either version 2.1 of the
License, or (at your option) any later version.
This library is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
USA
*/
package com.jstatcom.util;
import java.util.ArrayList;
import com.jstatcom.model.JSCDRange;
import com.jstatcom.model.JSCData;
import com.jstatcom.model.JSCNArray;
import com.jstatcom.model.JSCSArray;
import com.jstatcom.ts.TS;
import com.jstatcom.ts.TSDateRange;
import com.jstatcom.ts.TSHolder;
/**
* This class contains a collection of methods to be used with
* JSCData objects. Some methods are directly related to time
* series analysis. The methods defined here should help to prepare parameters
* for procedure calls, as well as to make simple data transformations often
* needed to prepare input panels. They are not used as often as to make them
* part of particular JSCData implementations.
*
* @author Markus Kraetzig
*/
public final class UData {
/**
* Not to be invoked.
*/
private UData() {
}
/**
* Creates a new data object with suffix being appended to
* all string elements in arg.
*
* @param arg
* the original data object
* @param suffix
* the string to be appended to all elements of arg
* @param newName
* the name of the new data object to be created
* @return a new data object with the changed string elements
*
* @throws IllegalArgumentException
* if (arg == null || suffix == null || newName == null)
* or if arg is not of a string type
*/
public static JSCSArray appendSuffix(JSCSArray arg, String suffix,
String newName) {
if (arg == null || suffix == null || newName == null)
throw new IllegalArgumentException("Argument was null.");
return new JSCSArray(newName, UStringArray.appendSuffix(arg
.stringArray(), suffix));
}
/**
* Creates a data object with the deterministic variables to be used for
* estimation according to the parametrization. The data object is
* constructed in the order
* dummies~intercept~seasonal dummies~trend, where each
* component may be there or not. If nothing is specified, an empty data
* object is returned.
*
* @param range
* the TSDateRange defining start and end
* @param dummies
* impulse and shift dummies, or other manually specified
* deterministics, can be either empty or null
* @param isIntercept
* true if a constant should be included
* @param isSeasDum
* true if seasonal dummies should be included,
* this automatically sets isIntercept=true as
* well, false otherwise
* @param isSeasDumCentered
* true if centered seasonal dummies should be
* used, only if (isSeasDum)
* @param isTrend
* true if a trend should be included
* @param newName
* the name of the new data object to create
*
* @return a new data object containing the deterministics for an estimation
* dummies~intercept~seasonal dummies~trend
*
* @throws IllegalArgumentException
* if (newName == null || range == null) or if
* the number of observations in dummies does not
* fit with range
*/
public static JSCNArray createDeterministic(TSDateRange range,
JSCNArray dummies, boolean isIntercept, boolean isSeasDum,
boolean isSeasDumCentered, boolean isTrend, String newName) {
if (dummies == null)
dummies = new JSCNArray("empty");
if (isSeasDum) // Always include an intercept if seasonal dummies are
// selected.
isIntercept = true;
if (newName == null || range == null)
throw new IllegalArgumentException("Argument was null.");
if (!dummies.isEmpty())
if (dummies.rows() != range.numOfObs())
throw new IllegalArgumentException(
"Wrong number of observations.");
JSCNArray data = new JSCNArray(newName);
// Append dummies.
if (!dummies.isEmpty())
data.appendCols(dummies);
// Append intercept.
if (isIntercept)
data.appendCols(new JSCNArray("intercept", UMatrix.ones(range
.numOfObs(), 1)));
// Append seasonal dummies.
if (isSeasDum)
data.appendCols(new JSCNArray("seasDum", range.createSeasDum(
isSeasDumCentered, false)));
// Append trend.
if (isTrend)
data.appendCols(new JSCNArray("trend", UMatrix.seqa(1, 1, range
.numOfObs())));
return data;
}
/**
* Creates a restriction matrix to be used for model estimation. The
* restriction matrix R is implicitely defined by vec(B)=R*gamma+r, where R
* is a (K(K*py + d + X(px+1)) x M) matrix with:
*
* - K being dimension of the process
*
- py the endogenous lags
*
- d the number of deterministic variables
*
- X the number of exogenous variables
*
- px the exogenous lags
*
- M the number of free parameters to estimate
*
- gamma (M x 1) vector of free parameters
*
- r vector of known constants, here assumed to be 0
*
- B matrix of all estimators
*
* See also H. Lutkepohl, Introduction to Multiple Time Series Analysis,
* 5.2.1. The suggested order of the restrictions can freely be set by
* rearranging the input data objects (exchanging det, end, ex).
* This method only supports exclusion restrictions for certain
* coefficients .
*
* @param det
* K x d matrix containing zeros and ones to exclude the
* respective coefficient for the deterministics
* @param end
* K x K*py matrix containing zeros and ones to exclude the
* respective coefficient for the endogenous part
* @param ex
* K x X(px+1) matrix containing zeros and ones to exclude the
* respective coefficient for the exogenous part
* @param name
* the name of the data object to create
*
* @return a new data object containing the restriction matrix, is empty if
* the input data objects are empty or if they do not contain any
* restrictions
*
* @throws IllegalArgumentException
* if (name == null || det == null || end == null || ex == null )
*/
public static JSCNArray createResMat(JSCNArray det, JSCNArray end,
JSCNArray ex, String name) {
if (name == null)
throw new IllegalArgumentException("Argument was null.");
JSCNArray res = new JSCNArray("merged");
if (det != null)
res.appendCols(det);
if (end != null)
res.appendCols(end);
if (ex != null)
res.appendCols(ex);
if (res.isEmpty())
return new JSCNArray(name);
int m = 0; // The number of free parameters to estimate.
ArrayList index = new ArrayList();
// Transform the restrictions and store indizes in a list.
for (int i = 0; i < res.cols(); i++)
for (int j = 0; j < res.rows(); j++)
if (res.intAt(j, i) == 1) {
m += 1;
index.add(new Integer(i * res.rows() + j));
}
// Initialize the (K(K*py + d + X(px+1)) x M) matrix.
double[][] restrictions = new double[res.cols() * res.rows()][m];
// Set the exclusion restrictions.
for (int i = 0; i < index.size(); i++)
restrictions[index.get(i).intValue()][i] = 1;
return new JSCNArray(name, restrictions);
}
/**
* Creates the Y and Z matrices to be used for estimation. For the
* definition of Z see Helmut Lutkepohl, Introduction to Multiple Time
* Series Analysis, 10.3. The input data objects must contain the same
* number of observations including the presample values which will be
* truncated according to the maximum number of endogenous/exogenous lags.
*
* The order of the variables within Z must be det|end|ex. If subset
* restrictions are set, then this must correspond to the ordering of R.
*
* The following symbols are defined:
*
* - T sample size
*
- D number of deterministic variables
*
- K number of endogenous variables (dimension of process)
*
- X number of deterministic variables
*
*
* @param detData
* data object with dimension
* T + Math.max(endLags, exLags) x D
* @param endData
* data object with dimension
* T + Math.max(endLags, exLags) x K, must not be
* empty
* @param exData
* data object with dimension
* T + Math.max(endLags, exLags) x X
* @param endLags
* number of endogenous lags in levels
* @param exLags
* number of exogenous lags in levels
* @param yName
* name of data object for Y matrix
* @param zName
* name of data object for Z matrix
*
* @return 2 x 1 array of data objects with the created matrices {Y, Z}
*
* @throws IllegalArgumentException
* if one of the arguments was null or
* if (endLags < 0 || exLags < 0) or
* if (endData.isEmpty()) or if the dimensions of
* the nonempty data objects are not the same or if there are
* too many lags for the given sample dize
*/
public static JSCNArray[] createYZMat(JSCNArray detData, JSCNArray endData,
JSCNArray exData, int endLags, int exLags, String yName,
String zName) {
if (detData == null || endData == null || exData == null
|| yName == null || zName == null)
throw new IllegalArgumentException("Argument was null.");
if (endLags < 0 || exLags < 0)
throw new IllegalArgumentException("Argument for laglength < 0.");
if (endData.isEmpty())
throw new IllegalArgumentException(
"No endogenous variables available.");
int k = endData.cols();
int x = exData.cols();
int d = detData.cols();
if (x == 0) // No exogenous variables, set lags to 0.
exLags = 0;
// Check the dimension of the data.
if ((d > 0 && detData.rows() != endData.rows())
|| (x > 0 && exData.rows() != endData.rows()))
throw new IllegalArgumentException(
"Number of observations incorrect.");
int maxLags = Math.max(endLags, exLags);
int rowsZ = k * endLags + x * (exLags + 1) + d;
int colsZ = endData.rows() - maxLags;
if (colsZ < 1)
throw new IllegalArgumentException(
"Too many lags for given sample (maxlags: " + maxLags
+ ", samplesize: " + endData.rows() + ").");
double Z[][] = new double[rowsZ][colsZ];
// Build the Z matrix.
for (int t = 0, ll = 0; t < colsZ; t++, ll = 0) {
for (int dd = 0; dd < d; dd++)
// include deterministic
Z[ll++][t] = detData.doubleAt(maxLags + t, dd);
for (int l = 0; l < endLags; l++)
// include endogenous
for (int kk = 0; kk < k; kk++)
Z[ll++][t] = endData.doubleAt(maxLags - 1 + t - l, kk);
for (int l = 0; l < exLags + 1; l++)
// include exogenous
for (int xx = 0; xx < x; xx++)
Z[ll++][t] = exData.doubleAt(maxLags + t - l, xx);
}
double Y[][] = UMatrix.getDoubleRows(endData.doubleArray(), maxLags,
endData.rows() - 1);
JSCNArray YZ[] = { new JSCNArray(yName, Y), new JSCNArray(zName, Z) };
return YZ;
}
/**
* Checks the rank of arg after deleting the first
* truncationLag rows of it. Returns whether the rank is
* equal to arg.getCols().
*
* @param arg
* the data object to be checked for column rank
* @param truncationLag
* the number of rows to delete from arg before
* checking the rank
* @return true if rank(arg) == >arg.getCols(),
* false otherwise
*
* @throws IllegalArgumentException
* if (arg == null) or
* if (truncationLag < 0) or
* if (arg.isEmpty())
*/
public static boolean hasFullColumnRank(JSCNArray arg, int truncationLag) {
if (arg == null)
throw new IllegalArgumentException("Argument was null.");
if (truncationLag < 0)
throw new IllegalArgumentException("Lag argument < 0.");
if (arg.isEmpty())
throw new IllegalArgumentException("Argument not Matrix.");
// Rough 1st check.
if (arg.rows() - truncationLag < arg.cols())
return false;
int rank = UMatrix.rank(UMatrix.getDoubleRows(arg.doubleArray(),
truncationLag, arg.rows() - 1));
return rank == arg.cols();
}
/**
* Gets the explicit representation of restrictions on the cointegrating
* space, given the implicit representation.
* Implicit Representation: res*vec(beta') = r
* Explicit Representation: vec(beta') =H*eta + h, with
* eta being the free parameters.
*
* In the context of cointegration analysis, beta is the part
* of the Beta matrix in Beta = [I_r : beta],
* because the first r columns of Beta are
* normalized, with r being the cointegration rank.
* eta are the elements of beta that are
* freely estimated after imposing the restrictions.
*
* @param res
* the matrix with implicit restrictions (J x K-r)
* @param r
* a vector with numbers (J x 1)
*
* @return a 3 x 1 array of data objects with H, a vector of
* zeros and ones selecting the free elements of vec(beta') and h,
* respectively
* @throws IllegalArgumentException
* if (res == null || r == null) or
* if (res.getRows() != r.getRows()) or if
* res contains restrictions that are linearly
* dependent
*/
public static JSCNArray[] imp2ExpRes(JSCNArray res, JSCNArray r) {
if (res == null || r == null)
throw new IllegalArgumentException("Argument was null.");
if (res.rows() != r.rows())
throw new IllegalArgumentException("Rows don't match.");
int rows = res.rows();
if (rows > 1) {
int rank = res.rank();
if (rank < res.rows())
throw new IllegalArgumentException(
"R does not have full row rank. Not all restrictions are independent.");
}
int cols = res.cols();
double[][] freeIndex = UMatrix.ones(cols, 2);
// Find free parameters.
for (int i = 0; i < rows; i++)
for (int j = 0; j < cols; j++) {
double val = res.doubleAt(i, j);
if (val != 0)
if (freeIndex[j][0] == 1) {
freeIndex[j][0] = 0;
freeIndex[j][1] = i;
// First modify R and r matrix such that cols that
// correspond to eliminated variables have
// only a single 1 and otherwise zeros in it (row
// echelon form but only for certain cols).
for (int k = 0; k < cols; k++)
// Divide row of R by val.
res.setValAt(res.doubleAt(i, k) / val, i, k);
// Divide element of r (right hand side) by val.
r.setValAt(r.doubleAt(i, 0) / val, i, 0);
// Now subtract multiple of normalized row from other
// rows.
for (int l = 0; l < rows; l++) {
if (res.doubleAt(l, j) != 0 && l != i) {
// Here another element in that column is
// nonzero.
val = res.doubleAt(l, j);
for (int k = 0; k < cols; k++)
res.setValAt(res.doubleAt(l, k) - val
* res.doubleAt(i, k), l, k);
// Also substract multiple on right hand side.
r.setValAt(r.doubleAt(l, 0) - val
* r.doubleAt(i, 0), l, 0);
}
}
break;
}
}
// Build H matrix only if there are any free parameters.
if (UMatrix.getNonzeroDoubleCount(UMatrix.getDoubleCol(freeIndex, 0)) == 0)
return new JSCNArray[] { new JSCNArray("HMat"),
new JSCNArray("eta", UMatrix.getDoubleCol(freeIndex, 0)),
new JSCNArray("h", r.doubleArray()), };
// Start from no restrictions.
double[][] hMat = UMatrix.eye(cols);
for (int i = 0; i < freeIndex.length; i++) {
if (freeIndex[i][0] == 0) {
int refIndex = (int) freeIndex[i][1];
// Express deleted variable with remaining parameters.
for (int j = 0; j < cols; j++) {
double val = res.doubleAt(refIndex, j);
hMat[i][j] = -1 * val;
}
}
}
// Delete columns that have no free parameters. Reduce counter by one,
// because hmat has one col less now.
for (int i = 0, k = 0; i < freeIndex.length; i++, k++)
if (freeIndex[i][0] == 0)
hMat = UMatrix.delCol(hMat, k--);
double[] h = new double[cols];
for (int i = 0; i < cols; i++)
if (freeIndex[i][0] == 0)
h[i] = r.doubleAt((int) freeIndex[i][1], 0);
return new JSCNArray[] { new JSCNArray("HMat", hMat),
new JSCNArray("eta", UMatrix.getDoubleCol(freeIndex, 0)),
new JSCNArray("h", h) };
}
/**
* Gets a new array with the elements of arg being
* transformed into an index with the mean of each column being normalized
* to +100 or -100, depending on the sign of the mean. This is usefull to
* compare series which differ greatly in levels. The index is computed as
* arg[i][j] * 100 / mean[j]
*
* @param arg
* the data object with the numbers to be transformed to an index
* @param newName
* the name of the new data object to be created
* @return a new data object with the indexed numbers
*
* @throws IllegalArgumentException
* if (arg == null || newName == null)
*/
public static JSCNArray indexData(JSCNArray arg, String newName) {
if (arg == null || newName == null)
throw new IllegalArgumentException("Argument was null.");
double[][] index = arg.doubleArray();
double[] mean = UMatrix.meanc(index);
for (int i = 0; i < arg.cols(); i++)
for (int j = 0; j < arg.rows(); j++)
index[j][i] = index[j][i] / Math.abs(mean[i]) * 100;
return new JSCNArray(newName, index);
}
/**
* Merges time series with possibly different length and fills them with
* missing values at the beginning and/or the end.
*
* @param namesOfData
* data object with the names of the time series in the
* TSHolder
* @param nameDat
* name of the new data object which holds the observations
* @param nameRange
* name of the new data object which holds the date range
*
* @return a 2 x 1 array of data objects, the 1st element holds the
* observations, the 2nd element holds the earliest date of all time
* series defined by namesOfData
* @throws IllegalArgumentException
* if (namesOfData == null || nameDat == null
* || nameRange == null)
*/
public static JSCData[] mergeTS(JSCSArray namesOfData, String nameDat,
String nameRange) {
if (namesOfData == null || nameDat == null || nameRange == null)
throw new IllegalArgumentException("Argument was null.");
TS[] sel = new TS[namesOfData.rows()];
for (int i = 0; i < sel.length; i++)
sel[i] = TSHolder.getInstance().getTS(namesOfData.stringAt(i, 0));
TSDateRange mergedRange = TSDateRange.getMergedRange(sel);
double[][] data = TS.getMergedData(sel);
JSCData[] ret = { new JSCNArray(nameDat, data),
new JSCDRange(nameRange, mergedRange) };
return ret;
}
/**
* Takes an array with deterministic terms and computes indices for the
* breaks of all dummy variables. Recognized variables are impulse, extended
* impulse, shift dummies, and trend breaks. All other variables are
* ignored.
*
* This method splits the data object with deterministic variables into the
* following data objects:
*
* - data object with indices of impulses for each impulse dummy with
* name "impulseIndices"
*
- data object with start and end indices of shifts with name
* "extImpulseIndices"
*
- data object with shift start indices with name "shiftIndices"
*
- data object with trend break start indices with name
* "trendBreakIndices"
*
* This method works only with impulse, extended impulse and shift dummies,
* not with seasonal summies, trend, or other deterministic variables. The
* following rules hold:
*
* - impulse dummy: 0001000 (there is a sequence 010)
*
- extended impulse dummy: 00111000 OR 11110000 (ends with 0 and has at
* least one 11 sequence)
*
- shift dummy: 00011111 (starts with 0, ends with 11 sequence)
*
- trended variables like 002345678
*
- dummy variables that contain only 11111, 0000, or a pure trend are
* ignored
*
- dummy variables like 001000100 will be indexed like 00111100
*
- all other variables are ignored
*
*
* @param dets
* the data object containing the deterministic variables
* @param zeroBased
* whether zero based index should be used, if false
* then 1 based index is used (like in Gauss)
* @return 4x1 array of data objects {impulse indices, extended impulse
* start/end indices, shift start indices, trend break indices}
* @throws IllegalArgumentException
* if (dets == null)
*/
public static JSCNArray[] splitDeterministics(JSCNArray dets,
boolean zeroBased) {
if (dets == null)
throw new IllegalArgumentException("Argument was null.");
JSCNArray impulseIndices = new JSCNArray("impulseIndices");
JSCNArray extImpulseIndices = new JSCNArray("extImpulseIndices");
JSCNArray shiftIndices = new JSCNArray("shiftIndices");
JSCNArray trendBreakIndices = new JSCNArray("trendBreakIndices");
// No dummies specified.
if (dets.isEmpty())
return new JSCNArray[] { impulseIndices, extImpulseIndices,
shiftIndices, trendBreakIndices };
double[][] detData = dets.doubleArray();
for (int i = 0; i < detData[0].length; i++) {
double[] coli = UMatrix.getDoubleCol(detData, i);
int numTest = UMatrix.getNonzeroDoubleCount(coli);
// Exclude only 0's or 1's or trend
if (numTest > 0 && numTest < detData.length) {
int firstNonZero = UMatrix.getFirstNonzeroIndex(coli);
int lastNonZero = UMatrix.getLastNonzeroIndex(coli);
// Impulse, has only one nonzero element
if (numTest == 1)
impulseIndices
.appendRows(new JSCNArray("impulseIndices",
new double[] { firstNonZero
+ (zeroBased ? 0 : 1) }));
// Shift, ends with nonzero, all
// elements same (checked only via bounds)
else if (coli[firstNonZero] == coli[coli.length - 1])
shiftIndices
.appendRows(new JSCNArray("shiftIndices",
new double[] { firstNonZero
+ (zeroBased ? 0 : 1) }));
// Extended Impulse, more than 1 element, ends with 0, all
// elements same (checked only via bounds)
else if (coli[firstNonZero] == coli[lastNonZero])
extImpulseIndices.appendRows(new JSCNArray(
"extImpulseIndices", new double[][] { {
firstNonZero + (zeroBased ? 0 : 1),
lastNonZero + (zeroBased ? 0 : 1) } }));
// Trend break if ends with nonzero which is different from
// first nonzero element
else if (coli[firstNonZero] != coli[coli.length - 1]
&& coli[coli.length - 1] != 0)
trendBreakIndices.appendRows(new JSCNArray(
"trendBreakIndices", new double[] { firstNonZero
+ (zeroBased ? 0 : 1) }));
}
}
return new JSCNArray[] { impulseIndices, extImpulseIndices,
shiftIndices, trendBreakIndices };
}
/**
* Creates a new data object with the elements of dat divided
* by the standard devitation of the respective columns of dat.
*
* @param dat
* the original data object
* @param name
* the name of the new data object to create
* @return a new data object with all elements standardized
*
* @throws IllegalArgumentException
* if (dat == null || name == null)
*/
public static JSCNArray standardize(JSCNArray dat, String name) {
if (dat == null || name == null)
throw new IllegalArgumentException("Argument was null.");
return new JSCNArray(name, UMatrix.standardize(dat.doubleArray()));
}
/**
* String representation of the first row of a string data object. The
* elements are just printed space separated.
*
* @param data
* array containing names
* @return string with names, for example "y x invest cons"
* @throws IllegalArgumentException
* if (array == null)
*/
public static String stringForArray(JSCSArray data) {
if (data == null)
throw new IllegalArgumentException(
"Argument was null for stringForArray.");
String ret = "";
for (int i = 0; i < data.rows(); i++)
ret = ret + data.stringAt(i, 0) + " ";
return ret.trim();
}
}
Ads help maintain this website.