From 16bafab54a41d29ae1bb349c2c5d22b46be10ba0 Mon Sep 17 00:00:00 2001 From: alexz Date: Mon, 12 Oct 2026 12:58:02 -0400 Subject: [PATCH] upload current sources --- .gitignore | 53 ++++ pom.xml | 78 +++++ .../minicircuits/AuthenticationService.java | 274 ++++++++++++++++++ .../minicircuits/BufferedResponseWrapper.java | 144 +++++++++ .../minicircuits/CharacterEncodingFilter.java | 35 +++ .../com/minicircuits/DatabaseManager.java | 163 +++++++++++ src/main/java/com/minicircuits/FetchAPI.java | 193 ++++++++++++ .../java/com/minicircuits/QueryResult.java | 63 ++++ .../java/com/minicircuits/StyleManager.java | 127 ++++++++ src/main/java/com/minicircuits/Utils.java | 178 ++++++++++++ .../minicircuits/formatter/CsvFormatter.java | 105 +++++++ .../formatter/ErrorFormatter.java | 71 +++++ .../formatter/ExcelFormatter.java | 135 +++++++++ .../formatter/FormatterFactory.java | 31 ++ .../minicircuits/formatter/HtmlFormatter.java | 107 +++++++ .../minicircuits/formatter/JsonFormatter.java | 88 ++++++ .../formatter/ResponseFormatter.java | 10 + .../minicircuits/formatter/RestFormatter.java | 156 ++++++++++ .../formatter/SuccessFormatter.java | 67 +++++ .../minicircuits/formatter/XmlFormatter.java | 86 ++++++ src/main/webapp/WEB-INF/web.xml | 31 ++ 21 files changed, 2195 insertions(+) create mode 100644 .gitignore create mode 100644 pom.xml create mode 100644 src/main/java/com/minicircuits/AuthenticationService.java create mode 100644 src/main/java/com/minicircuits/BufferedResponseWrapper.java create mode 100644 src/main/java/com/minicircuits/CharacterEncodingFilter.java create mode 100644 src/main/java/com/minicircuits/DatabaseManager.java create mode 100644 src/main/java/com/minicircuits/FetchAPI.java create mode 100644 src/main/java/com/minicircuits/QueryResult.java create mode 100644 src/main/java/com/minicircuits/StyleManager.java create mode 100644 src/main/java/com/minicircuits/Utils.java create mode 100644 src/main/java/com/minicircuits/formatter/CsvFormatter.java create mode 100644 src/main/java/com/minicircuits/formatter/ErrorFormatter.java create mode 100644 src/main/java/com/minicircuits/formatter/ExcelFormatter.java create mode 100644 src/main/java/com/minicircuits/formatter/FormatterFactory.java create mode 100644 src/main/java/com/minicircuits/formatter/HtmlFormatter.java create mode 100644 src/main/java/com/minicircuits/formatter/JsonFormatter.java create mode 100644 src/main/java/com/minicircuits/formatter/ResponseFormatter.java create mode 100644 src/main/java/com/minicircuits/formatter/RestFormatter.java create mode 100644 src/main/java/com/minicircuits/formatter/SuccessFormatter.java create mode 100644 src/main/java/com/minicircuits/formatter/XmlFormatter.java create mode 100644 src/main/webapp/WEB-INF/web.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f41272b --- /dev/null +++ b/.gitignore @@ -0,0 +1,53 @@ +# added by gitlab-sync +.DS_Store +Thumbs.db +desktop.ini +.idea/ +*.iml +.vscode/ +*.swp +*~ +target/ +build/ +dist/ +out/ +*.class +.gradle/ +node_modules/ +__pycache__/ +*.pyc +.venv/ +venv/ +coverage/ +*.log +*.tmp +.env +.env.* +!.env.example +.claude/ +CLAUDE.md +CLAUDE.local.md +.mcp.json +.remember/ +.cursor/ +.cursorrules +.cursorignore +.windsurf/ +.windsurfrules +.codeium/ +.github/copilot-instructions.md +.copilot/ +.codex/ +AGENTS.md +.gemini/ +GEMINI.md +.aider* +.continue/ +.roo/ +.roomodes +.cline/ +.clinerules +.kiro/ +.amazonq/ +.tabnine/ +.specstory/ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..c6ae9c7 --- /dev/null +++ b/pom.xml @@ -0,0 +1,78 @@ + + 4.0.0 + + com.minicircuits + fetch-api + 1.0.0 + war + FetchAPI + API for fetching data from AS400 database + + + UTF-8 + 11 + 11 + 5.2.3 + 10.7 + 20231013 + 4.0.1 + + + + + net.sf.jt400 + jt400 + ${jt400.version} + + + org.apache.poi + poi + ${poi.version} + + + org.apache.poi + poi-ooxml + ${poi.version} + + + org.json + json + ${json.version} + + + javax.servlet + javax.servlet-api + ${servlet.version} + provided + + + javax.xml.bind + jaxb-api + 2.3.1 + + + + FetchAPI + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + ${maven.compiler.source} + ${maven.compiler.target} + + + + org.apache.maven.plugins + maven-war-plugin + 3.3.2 + + false + + + + + diff --git a/src/main/java/com/minicircuits/AuthenticationService.java b/src/main/java/com/minicircuits/AuthenticationService.java new file mode 100644 index 0000000..e5f2060 --- /dev/null +++ b/src/main/java/com/minicircuits/AuthenticationService.java @@ -0,0 +1,274 @@ +package com.minicircuits; + +import com.ibm.as400.access.AS400; +import com.ibm.as400.access.AS400SecurityException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.json.JSONObject; + +public class AuthenticationService { + private static final Logger logger = Logger.getLogger(AuthenticationService.class.getName()); + private static final String SESSION_COOKIE_NAME = "FETCHAPI_SESSION"; + private static final int CLEANUP_THRESHOLD = 1000; + + private final Properties configProps; + private final Properties APITokens; + private final ConcurrentHashMap sessionCache = new ConcurrentHashMap<>(); + private final int ttlMinutes; + + private static class SessionInfo { + final String userId; + final long expiresAt; + final String hmacSecret; + + SessionInfo(String userId, long expiresAt, String hmacSecret) { + this.userId = userId; + this.expiresAt = expiresAt; + this.hmacSecret = hmacSecret; + } + + boolean isExpired() { + return System.currentTimeMillis() > expiresAt; + } + } + + public AuthenticationService(Properties configProps, Properties APITokens) { + this.configProps = configProps; + this.APITokens = APITokens; + int ttl; + try { + ttl = Integer.parseInt(configProps.getProperty("session.ttl.minutes", "30")); + } catch (NumberFormatException e) { + ttl = 30; + } + this.ttlMinutes = ttl; + + logger.log(Level.INFO, "Session TTL configured to {0} minutes", this.ttlMinutes); + } + + /** + * Backward-compatible overload for callers that do not have an HttpServletResponse. + * No session cookie will be issued, but existing auth logic still works. + */ + public boolean authenticate(HttpServletRequest req) { + return authenticate(req, null); + } + + public boolean authenticate(HttpServletRequest req, HttpServletResponse resp) { + // Lazy cleanup when the cache grows large + if (sessionCache.size() > CLEANUP_THRESHOLD) { + cleanupExpiredSessions(); + } + + // 1. Check for existing session cookie + String existingToken = getSessionCookie(req); + if (existingToken != null) { + SessionInfo cachedSession = sessionCache.get(existingToken); + if (cachedSession != null && !cachedSession.isExpired()) { + // Verify HMAC with the cached secret + if (verifyToken(existingToken, cachedSession.hmacSecret)) { + logger.log(Level.FINE, "Verified session for user: {0}", cachedSession.userId); + return true; + } + } + sessionCache.remove(existingToken); + } + + // 2. Fall back to credential / API-token authentication + String userId = req.getParameter("userId"); + String password = req.getParameter("password"); + String apiToken = req.getParameter("apiToken"); + + if ((userId == null || password == null) && apiToken == null) { + logger.warning("No credentials provided"); + return false; + } + + boolean authenticated = userId != null && password != null + ? authenticateWithCredentials(userId, password) + : apiToken != null && validateApiToken(apiToken); + + // 3. On success, create session and set cookie + if (authenticated && resp != null) { + String identity = (userId != null) ? userId : "apiToken"; + // Use credentials as HMAC key: userId:password for credentials, raw API token for token auth + String hmacKey = (userId != null && password != null) + ? userId + ":" + password + : apiToken; + String sessionToken = createSession(identity, hmacKey); + resp.setHeader("Set-Cookie", SESSION_COOKIE_NAME + "=" + sessionToken + + "; HttpOnly; Secure; SameSite=Strict; Path=/; Max-Age=" + (ttlMinutes * 60)); + logger.log(Level.INFO, "Session created for: {0}", identity); + } + + return authenticated; + } + + public boolean authenticateWithCredentials(String userId, String password) { + AS400 system = null; + try { + system = new AS400( + configProps.getProperty("db.server"), + userId, + password + ); + system.validateSignon(); + logger.log(Level.INFO, "User authenticated successfully: {0}", userId); + return true; + } catch (AS400SecurityException | IOException e) { + logger.log(Level.WARNING, "Authentication failed", e); + return false; + } finally { + if (system != null) { + system.disconnectAllServices(); + } + } + } + + private String getSessionCookie(HttpServletRequest req) { + Cookie[] cookies = req.getCookies(); + if (cookies != null) { + for (Cookie cookie : cookies) { + if (SESSION_COOKIE_NAME.equals(cookie.getName())) { + return cookie.getValue(); + } + } + } + return null; + } + + private String createSession(String userId, String hmacSecret) { + long createdAt = System.currentTimeMillis(); + long expiresAt = createdAt + ((long) ttlMinutes * 60 * 1000); + String serverName = configProps.getProperty("db.server", "localhost"); + + // HMAC signs the core fields + String hmacData = userId + ":" + serverName + ":" + createdAt + ":" + expiresAt; + String signature = hmacSha256(hmacData, hmacSecret); + + // Build JSON payload + JSONObject tokenJson = new JSONObject(); + tokenJson.put("userId", userId); + tokenJson.put("serverName", serverName); + tokenJson.put("createdAt", createdAt); + tokenJson.put("expiresAt", expiresAt); + tokenJson.put("signature", signature); + + // Base64 encode the entire JSON string + String token = Base64.getUrlEncoder().withoutPadding().encodeToString( + tokenJson.toString().getBytes(StandardCharsets.UTF_8)); + + sessionCache.put(token, new SessionInfo(userId, expiresAt, hmacSecret)); + return token; + } + + private String hmacSha256(String data, String key) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + SecretKeySpec secretKey = new SecretKeySpec( + key.getBytes(StandardCharsets.UTF_8), "HmacSHA256"); + mac.init(secretKey); + byte[] hmacBytes = mac.doFinal(data.getBytes(StandardCharsets.UTF_8)); + return Base64.getUrlEncoder().withoutPadding().encodeToString(hmacBytes); + } catch (NoSuchAlgorithmException | InvalidKeyException e) { + throw new RuntimeException("Failed to compute HMAC", e); + } + } + + private boolean verifyToken(String base64Token, String hmacSecret) { + try { + String decoded = new String(Base64.getUrlDecoder().decode(base64Token), StandardCharsets.UTF_8); + JSONObject tokenJson = new JSONObject(decoded); + + String userId = tokenJson.getString("userId"); + String serverName = tokenJson.getString("serverName"); + long createdAt = tokenJson.getLong("createdAt"); + long expiresAt = tokenJson.getLong("expiresAt"); + String signature = tokenJson.getString("signature"); + + // Check expiry + if (System.currentTimeMillis() > expiresAt) return false; + + // Verify server + String currentServer = configProps.getProperty("db.server", "localhost"); + if (!serverName.equals(currentServer)) return false; + + // Verify HMAC + String hmacData = userId + ":" + serverName + ":" + createdAt + ":" + expiresAt; + String expectedSignature = hmacSha256(hmacData, hmacSecret); + + return MessageDigest.isEqual( + signature.getBytes(StandardCharsets.UTF_8), + expectedSignature.getBytes(StandardCharsets.UTF_8)); + } catch (IllegalArgumentException | org.json.JSONException e) { + logger.warning("Failed to parse session token"); + return false; + } + } + + private void cleanupExpiredSessions() { + int removed = 0; + Iterator> it = sessionCache.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + if (entry.getValue().isExpired()) { + it.remove(); + removed++; + } + } + if (removed > 0) { + logger.log(Level.INFO, "Cleaned up {0} expired sessions, {1} remaining", + new Object[]{removed, sessionCache.size()}); + } + } + + public boolean validateApiToken(String tokenToValidate) { + try { + String decodedToken = Utils.decodeBase64(tokenToValidate); + String[] parts = decodedToken.split(":"); + + if (parts.length != 2) { + logger.warning("Invalid token format"); + return false; + } + + String identifier = parts[0]; + String uuid = parts[1]; + String validToken = APITokens.getProperty(identifier); + + if (validToken == null || validToken.trim().isEmpty()) { + logger.log(Level.WARNING, "No API token configured for: {0}", identifier); + return false; + } + + if (validToken.equals(uuid)) { + logger.log(Level.INFO, "API token validated for: {0}", identifier); + return true; + } + + logger.log(Level.WARNING, "Invalid token attempted for: {0}", identifier); + return false; + } catch (IllegalArgumentException e) { + logger.log(Level.SEVERE, "Invalid Base64 in API token", e); + return false; + } catch (Exception e) { + logger.log(Level.SEVERE, "Error validating API token", e); + return false; + } + } +} diff --git a/src/main/java/com/minicircuits/BufferedResponseWrapper.java b/src/main/java/com/minicircuits/BufferedResponseWrapper.java new file mode 100644 index 0000000..b8e2974 --- /dev/null +++ b/src/main/java/com/minicircuits/BufferedResponseWrapper.java @@ -0,0 +1,144 @@ +package com.minicircuits; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; + +import javax.servlet.ServletOutputStream; +import javax.servlet.WriteListener; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponseWrapper; + +public class BufferedResponseWrapper extends HttpServletResponseWrapper { + private final Path tempFile; + private final FileOutputStream fileOutputStream; + private final ServletOutputStream servletOutputStream; + private PrintWriter writer; + + public BufferedResponseWrapper(HttpServletResponse response) throws IOException { + super(response); + this.tempFile = Files.createTempFile("fetchapi_", ".tmp"); + this.fileOutputStream = new FileOutputStream(tempFile.toFile()); + this.servletOutputStream = new ServletOutputStream() { + @Override + public void write(int b) throws IOException { + fileOutputStream.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + fileOutputStream.write(b, off, len); + } + + @Override + public void flush() throws IOException { + fileOutputStream.flush(); + } + + @Override + public void close() throws IOException { + fileOutputStream.close(); + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public void setWriteListener(WriteListener writeListener) { + // Not used for blocking I/O + } + }; + } + + @Override + public ServletOutputStream getOutputStream() { + return servletOutputStream; + } + + @Override + public PrintWriter getWriter() throws IOException { + if (writer == null) { + String charset = getCharacterEncoding(); + writer = new PrintWriter(new OutputStreamWriter(fileOutputStream, + charset != null ? charset : "UTF-8")); + } + return writer; + } + + @Override + public void flushBuffer() { + // Don't flush to the real response — we're buffering + } + + @Override + public void reset() { + // Don't reset the real response during buffering + } + + @Override + public void resetBuffer() { + // Don't reset during buffering + } + + /** + * Flush any remaining data, then stream the temp file to the real response. + * Sets Content-Length. Deletes the temp file when done. + */ + public void writeToResponse() throws IOException { + // Flush and close writer/stream + if (writer != null) { + writer.close(); + } else { + fileOutputStream.close(); + } + + // Set content length from temp file size + long fileSize = Files.size(tempFile); + if (fileSize <= Integer.MAX_VALUE) { + getResponse().setContentLength((int) fileSize); + } else { + ((HttpServletResponse) getResponse()).setHeader("Content-Length", Long.toString(fileSize)); + } + + // Stream temp file to real response + HttpServletResponse realResponse = (HttpServletResponse) getResponse(); + try (InputStream in = new BufferedInputStream(new FileInputStream(tempFile.toFile()), 32768); + OutputStream out = new BufferedOutputStream(realResponse.getOutputStream(), 32768)) { + byte[] buffer = new byte[32768]; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + out.flush(); + } finally { + // Always delete the temp file + Files.deleteIfExists(tempFile); + } + } + + /** + * Clean up temp file if writeToResponse was never called (e.g., on error). + */ + public void cleanup() { + try { + if (fileOutputStream != null) { + fileOutputStream.close(); + } + } catch (IOException ignored) { + } + try { + Files.deleteIfExists(tempFile); + } catch (IOException ignored) { + } + } +} diff --git a/src/main/java/com/minicircuits/CharacterEncodingFilter.java b/src/main/java/com/minicircuits/CharacterEncodingFilter.java new file mode 100644 index 0000000..49c78d6 --- /dev/null +++ b/src/main/java/com/minicircuits/CharacterEncodingFilter.java @@ -0,0 +1,35 @@ + +package com.minicircuits; + +import java.io.IOException; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; + +public class CharacterEncodingFilter implements Filter { + private String encoding; + + @Override + public void init(FilterConfig config) throws ServletException { + encoding = config.getInitParameter("encoding"); + if (encoding == null) { + encoding = "UTF-8"; + } + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) throws IOException, ServletException { + request.setCharacterEncoding(encoding); + response.setCharacterEncoding(encoding); + chain.doFilter(request, response); + } + + @Override + public void destroy() { + } +} \ No newline at end of file diff --git a/src/main/java/com/minicircuits/DatabaseManager.java b/src/main/java/com/minicircuits/DatabaseManager.java new file mode 100644 index 0000000..0e92b43 --- /dev/null +++ b/src/main/java/com/minicircuits/DatabaseManager.java @@ -0,0 +1,163 @@ +package com.minicircuits; + +import java.beans.PropertyVetoException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.ibm.as400.access.AS400JDBCConnectionPool; +import com.ibm.as400.access.AS400JDBCConnectionPoolDataSource; +import com.ibm.as400.access.ConnectionPoolException; + +public class DatabaseManager { + private static final String JDBC_DRIVER = "com.ibm.as400.access.AS400JDBCDriver"; + private static final Logger logger = Logger.getLogger(DatabaseManager.class.getName()); + + private AS400JDBCConnectionPool pool; + private final Properties configProps; + + public DatabaseManager(Properties configProps) throws Exception { + this.configProps = configProps; + Class.forName(JDBC_DRIVER); + initializeConnectionPool(); + } + + private void initializeConnectionPool() throws Exception { + try { + String username = configProps.getProperty("db.username"); + String password = Utils.decodeBase64(configProps.getProperty("db.password")); + String serverName = configProps.getProperty("db.server", "localhost"); + String jdbcString = configProps.getProperty("db.jdbcString"); + String libraries = configProps.getProperty("db.libraries", "QGPL,QTEMP"); + + validateConfig(username, password); + AS400JDBCConnectionPoolDataSource dataSource = createDataSource( + serverName, + username, + password, + jdbcString, + libraries + ); + pool = new AS400JDBCConnectionPool(dataSource); + configurePool(pool); + } catch (PropertyVetoException | ConnectionPoolException e) { + throw new Exception("Failed to initialize connection pool", e); + } + } + + private void validateConfig(String username, String password) throws Exception { + if (username == null || password == null) { + throw new Exception("Database username or password not provided in properties file"); + } + } + + private AS400JDBCConnectionPoolDataSource createDataSource( + String serverName, + String username, + String password, + String jdbcString, + String libraries + ) throws PropertyVetoException { + AS400JDBCConnectionPoolDataSource dataSource = new AS400JDBCConnectionPoolDataSource(); + dataSource.setServerName(serverName); + dataSource.setUser(username); + dataSource.setPassword(password); + dataSource.setProperties(jdbcString); + dataSource.setLibraries(libraries); + return dataSource; + } + + private void configurePool(AS400JDBCConnectionPool pool) throws ConnectionPoolException { + pool.setMaxConnections(40); + pool.fill(5); + } + + public QueryResult executeQuery(String sqlQuery) throws SQLException, ConnectionPoolException { + // Log connection pool status + logger.log(Level.INFO, "Connection pool before query - Active connections: {0}, Available connections: {1}", + new Object[]{pool.getActiveConnectionCount(), pool.getAvailableConnectionCount()}); + + Connection connection = pool.getConnection(); + logger.log(Level.INFO, "Got new connection from pool: {0}", connection.hashCode()); + + PreparedStatement statement = null; + ResultSet resultSet = null; + + try { + // Set up library list + try (Statement libStmt = connection.createStatement()) { + libStmt.execute("Values Amulibq.Set_Library_List()"); + } + + statement = connection.prepareStatement( + sqlQuery, + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY + ); + + statement.setFetchSize(5000); + statement.setQueryTimeout(Integer.parseInt( + configProps.getProperty("query.timeout.seconds", "600") + )); + + // Execute the statement and check if it returns a ResultSet + boolean hasResultSet = statement.execute(); + + if (hasResultSet) { + // Handle SELECT queries + resultSet = statement.getResultSet(); + String[] headers = getQueryHeaders(resultSet); + + return new QueryResult( + connection, + statement, + resultSet, + headers, + resultSet.getMetaData() + ); + } else { + // Handle non-SELECT queries (CREATE, INSERT, etc.) + return new QueryResult( + connection, + statement, + null, + new String[0], + null + ); + } + } catch (SQLException e) { + // Clean up resources in case of error + if (resultSet != null) resultSet.close(); + if (statement != null) statement.close(); + connection.close(); + throw e; + } + } + + private String[] getQueryHeaders(ResultSet rs) throws SQLException { + ResultSetMetaData rsmd = rs.getMetaData(); + int columnCount = rsmd.getColumnCount(); + String[] headers = new String[columnCount]; + + for (int i = 1; i <= columnCount; i++) { + String label = rsmd.getColumnLabel(i); + headers[i - 1] = (label != null && !label.trim().isEmpty()) + ? Utils.toTitleCase(label.trim()) + : Utils.toTitleCase(rsmd.getColumnName(i)); + } + return headers; + } + + public void closePool() { + if (pool != null) { + pool.close(); + logger.info("Connection pool closed."); + } + } +} diff --git a/src/main/java/com/minicircuits/FetchAPI.java b/src/main/java/com/minicircuits/FetchAPI.java new file mode 100644 index 0000000..5df23f4 --- /dev/null +++ b/src/main/java/com/minicircuits/FetchAPI.java @@ -0,0 +1,193 @@ +package com.minicircuits; + +import com.minicircuits.formatter.*; +import java.io.*; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import com.ibm.as400.access.ConnectionPoolException; +import javax.servlet.*; +import javax.servlet.http.*; + +public class FetchAPI extends HttpServlet { + + private static final Logger logger = Logger.getLogger(FetchAPI.class.getName()); + + private static final Pattern DANGEROUS_SQL = Pattern.compile( + "\\b(TRUNCATE|DROP\\s+(TABLE|VIEW|INDEX|SCHEMA|FUNCTION|PROCEDURE|TRIGGER|SEQUENCE|ALIAS|TYPE|VARIABLE|PACKAGE)|DELETE\\s+FROM|ALTER|GRANT|REVOKE|RENAME)\\b", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL + ); + + private DatabaseManager dbManager; + private AuthenticationService authService; + private Properties configProps; + private Properties APITokens; + private ErrorFormatter errorFormatter; + private SuccessFormatter successFormatter; + + @Override + public void init() throws ServletException { + System.setProperty( + "org.apache.logging.log4j.simplelog.StatusLogger.level", + "OFF" + ); + try { + configProps = loadProperties("/etc/misc/configs/FetchAPI.properties"); + APITokens = loadProperties("/etc/misc/configs/APITokens.properties"); + + dbManager = new DatabaseManager(configProps); + authService = new AuthenticationService(configProps, APITokens); + errorFormatter = FormatterFactory.createErrorFormatter(); + successFormatter = FormatterFactory.createSuccessFormatter(); + + logger.info("FetchAPI initialized successfully"); + } catch (Exception e) { + logger.log(Level.SEVERE, "Failed to initialize FetchAPI", e); + throw new ServletException(e); + } + } + + private Properties loadProperties(String propsPath) throws ServletException { + Properties props = new Properties(); + try (FileInputStream in = new FileInputStream(propsPath)) { + props.load(in); + return props; + } catch (IOException e) { + throw new ServletException("Failed to load properties file", e); + } + } + + @Override + protected void doGet( + HttpServletRequest request, + HttpServletResponse response + ) throws ServletException, IOException { + processRequest(request, response); + } + + @Override + protected void doPost( + HttpServletRequest request, + HttpServletResponse response + ) throws ServletException, IOException { + processRequest(request, response); + } + + private String getOutputFormat(HttpServletRequest request) { + String format = request.getParameter("format"); + if (format == null || format.isBlank()) { + String acceptHeader = request.getHeader("Accept"); + if (acceptHeader != null) { + if (acceptHeader.contains("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) { + return "excel"; + } else if (acceptHeader.contains("text/csv")) { + return "csv"; + } else if (acceptHeader.equals("application/xml") || acceptHeader.equals("text/xml")) { + return "xml"; + } else if (acceptHeader.equals("text/html")) { + return "html"; + } + } + return "json"; // default + } + return format.toLowerCase(); + } + + private void processRequest( + HttpServletRequest request, + HttpServletResponse response + ) throws ServletException, IOException { + QueryResult result = null; + request.setCharacterEncoding("UTF-8"); + Utils.addCorsHeaders(response); + + String format = getOutputFormat(request); + + if (!authService.authenticate(request, response)) { + errorFormatter.format( + response, + HttpServletResponse.SC_UNAUTHORIZED, + "Not Authorized!", + format + ); + return; + } + + // Works for both GET query params and POST form data + String sqlQuery = request.getParameter("sql"); + if (!validateQuery(sqlQuery)) { + errorFormatter.format( + response, + HttpServletResponse.SC_BAD_REQUEST, + "Invalid or dangerous SQL query", + format + ); + return; + } + + try { + logger.log(Level.INFO, "Executing query: {0}", sqlQuery); + result = dbManager.executeQuery(sqlQuery); + + if (result.getResultSet() == null) { + // Non-query operation — write directly, no temp file needed + successFormatter.format(response, "SQL executed successfully", format); + logger.info("Non-query SQL executed successfully"); + } else { + // Query operation — buffer to temp file, then serve + BufferedResponseWrapper bufferedResponse = new BufferedResponseWrapper(response); + try { + ResponseFormatter formatter = FormatterFactory.getFormatter(format, configProps); + formatter.format(bufferedResponse, result); + bufferedResponse.writeToResponse(); + logger.info("Query results formatted and served successfully"); + } catch (IOException | java.sql.SQLException | RuntimeException e) { + bufferedResponse.cleanup(); + throw e; + } + } + + } catch (IOException | RuntimeException | java.sql.SQLException | ConnectionPoolException e) { + logger.log(Level.SEVERE, "Error processing request", e); + if (!response.isCommitted()) { + errorFormatter.format( + response, + HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "An internal error occurred while processing your request", + format + ); + } + } finally { + if (result != null) { + try { + result.close(); + logger.info("Result closed"); + } catch (Exception e) { + logger.log(Level.WARNING, "Error closing database resources", e); + } + } + } + } + + private boolean validateQuery(String sqlQuery) { + if (sqlQuery == null || sqlQuery.isBlank()) { + return false; + } + // Strip comments and string literals so they can't hide keywords + String stripped = sqlQuery + .replaceAll("(?s)/\\*.*?\\*/", " ") // block comments + .replaceAll("--[^\n]*", " ") // line comments + .replaceAll("'[^']*'", "''"); // string literals + + return !DANGEROUS_SQL.matcher(stripped).find(); + } + + @Override + public void destroy() { + if (dbManager != null) { + dbManager.closePool(); + logger.info("Database connection pool closed."); + } + } +} diff --git a/src/main/java/com/minicircuits/QueryResult.java b/src/main/java/com/minicircuits/QueryResult.java new file mode 100644 index 0000000..f00ba14 --- /dev/null +++ b/src/main/java/com/minicircuits/QueryResult.java @@ -0,0 +1,63 @@ +package com.minicircuits; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +public class QueryResult implements AutoCloseable { + + public final ResultSet resultSet; + public final String[] headers; + public final ResultSetMetaData metadata; + public final Connection connection; + public final PreparedStatement statement; + + public QueryResult( + Connection conn, + PreparedStatement stmt, + ResultSet rs, + String[] headers, + ResultSetMetaData metadata + ) { + this.connection = conn; + this.statement = stmt; + this.resultSet = rs; + this.headers = headers; + this.metadata = metadata; + } + + public ResultSet getResultSet() { + return resultSet; + } + + public String[] getHeaders() { + return headers; + } + + public ResultSetMetaData getMetadata() { + return metadata; + } + + public Connection getConnection() { + return connection; + } + + public PreparedStatement getStatement() { + return statement; + } + + @Override + public void close() throws Exception { + try { + if (resultSet != null) resultSet.close(); + } finally { + try { + if (statement != null) statement.close(); + } finally { + if (connection != null) connection.close(); + } + } + } +} \ No newline at end of file diff --git a/src/main/java/com/minicircuits/StyleManager.java b/src/main/java/com/minicircuits/StyleManager.java new file mode 100644 index 0000000..d0ff7a0 --- /dev/null +++ b/src/main/java/com/minicircuits/StyleManager.java @@ -0,0 +1,127 @@ +package com.minicircuits; + +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.poi.ss.usermodel.*; + +public class StyleManager { + private final Map styleCache = new ConcurrentHashMap<>(); + private final Properties configProps; + + public StyleManager(Properties configProps) { + this.configProps = configProps; + } + + public CellStyle createHeaderStyle(Workbook workbook) { + CellStyle headerStyle = workbook.createCellStyle(); + headerStyle.setFillForegroundColor(IndexedColors.ROYAL_BLUE.getIndex()); + headerStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); + headerStyle.setWrapText(true); + + Font headerFont = workbook.createFont(); + headerFont.setColor(IndexedColors.WHITE.getIndex()); + headerFont.setBold(true); + headerStyle.setFont(headerFont); + + return headerStyle; + } + + public CellStyle getCellStyle(String styleType, Workbook workbook) { + return styleCache.computeIfAbsent(styleType, type -> createCellStyle(type, workbook)); + } + + private CellStyle createCellStyle(String styleType, Workbook workbook) { + CellStyle style = workbook.createCellStyle(); + + // Set alignment + String alignmentProp = configProps.getProperty("style." + styleType + ".alignment", "LEFT"); + HorizontalAlignment alignment = HorizontalAlignment.valueOf(alignmentProp); + style.setAlignment(alignment); + + // Set format if exists + String format = configProps.getProperty("style." + styleType + ".format"); + if (format != null) { + DataFormat dataFormat = workbook.createDataFormat(); + style.setDataFormat(dataFormat.getFormat(format)); + } + + return style; + } + + public void setColumnStyles(Sheet sheet, ResultSetMetaData metadata) throws SQLException { + for (int i = 1; i <= metadata.getColumnCount(); i++) { + int sqlType = metadata.getColumnType(i); + int scale = metadata.getScale(i); + + String styleType = determineStyleType(sqlType, scale); + CellStyle style = getCellStyle(styleType, sheet.getWorkbook()); + sheet.setDefaultColumnStyle(i - 1, style); + } + } + + public String determineStyleType(int sqlType, int scale) throws SQLException { + switch (sqlType) { + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + return "text"; + + case Types.DECIMAL: + case Types.NUMERIC: + return scale > 0 ? "number" : "integer"; + + case Types.DOUBLE: + case Types.FLOAT: + case Types.REAL: + return "number"; // Always decimal for floating point types + + case Types.BIGINT: + case Types.INTEGER: + case Types.SMALLINT: + case Types.TINYINT: + return "integer"; + + case Types.DATE: + return "date"; + + case Types.TIMESTAMP: + return "timestamp"; + + default: + return "text"; + } + } + + public void setOptimalColumnWidths(Sheet sheet, String[] headers, ResultSetMetaData metadata) throws SQLException { + for (int i = 0; i < headers.length; i++) { + int columnWidth = metadata.getColumnDisplaySize(i + 1); + int excelWidth = getOptimalColumnLength(headers[i], columnWidth); + sheet.setColumnWidth(i, excelWidth); + } + } + + private int getOptimalColumnLength(String header, int dataLength) { + final int DEFAULT_MAX = Integer.parseInt( + configProps.getProperty("excel.column.max.width", "40") + ); + final int DEFAULT_MIN = Integer.parseInt( + configProps.getProperty("excel.column.min.width", "10") + ); + int maxLength = 0; + + for (String word : header.split("\\s+")) { + maxLength = Math.max(maxLength, word.length()); + } + maxLength = Math.max(maxLength, dataLength); + maxLength = Math.max(DEFAULT_MIN, Math.min(maxLength, DEFAULT_MAX)); + return maxLength * 256; + } + + public void clearCache() { + styleCache.clear(); + } +} \ No newline at end of file diff --git a/src/main/java/com/minicircuits/Utils.java b/src/main/java/com/minicircuits/Utils.java new file mode 100644 index 0000000..bac0afc --- /dev/null +++ b/src/main/java/com/minicircuits/Utils.java @@ -0,0 +1,178 @@ +package com.minicircuits; + +import java.nio.charset.StandardCharsets; +import java.sql.Types; +import java.util.Base64; +import javax.servlet.http.HttpServletResponse; + +public class Utils { + + public static String toTitleCase(String input) { + if (input == null || input.isEmpty()) { + return input; + } + + String[] words = (input.replaceAll("\\s+", " ").trim()).split("[ _]"); + StringBuilder result = new StringBuilder(); + + for (String word : words) { + if (!word.isEmpty()) { + result + .append(Character.toUpperCase(word.charAt(0))) + .append(word.substring(1).toLowerCase()) + .append(" "); + } + } + return result.toString().trim(); + } + + public static String rightTrim(String s) { + int i = s.length() - 1; + while (i >= 0 && Character.isWhitespace(s.charAt(i))) { + i--; + } + return s.substring(0, i + 1); + } + + public static String decodeBase64(String encodedString) { + return new String( + Base64.getDecoder().decode(encodedString), + StandardCharsets.UTF_8 + ); + } + + public static String escapeHtml(String input) { + if (input == null) return ""; + return input + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """) + .replace("'", "'"); + } + + public static String escapeXml(String input) { + if (input == null) return ""; + return input + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """) + .replace("'", "'"); + } + + public static void addCorsHeaders(HttpServletResponse resp) { + resp.setHeader("Access-Control-Allow-Origin", "*"); + resp.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + resp.setHeader("Access-Control-Allow-Headers", "Content-Type"); + resp.setHeader("Access-Control-Max-Age", "3600"); + resp.setHeader("Connection", "close"); + resp.setHeader("Keep-Alive", "timeout=0, max=0"); + resp.setHeader("Pragma", "no-cache"); + resp.setHeader("Cache-Control", "no-cache, no-store, must-revalidate"); + + // Add a unique timestamp to prevent caching + resp.setHeader("X-Timestamp", String.valueOf(System.currentTimeMillis())); + } + + public static String escapeJson(String s) { + if (s == null) return ""; + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '"': sb.append("\\\""); break; + case '\\': sb.append("\\\\"); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: + if (c < 0x20) { + sb.append(String.format("\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + } + return sb.toString(); + } + + /** + * "CUSTOMER_NAME" -> "customerName" + * "Customer Name" -> "customerName" + * "CUSTNAME" -> "custname" + */ + public static String toCamelCase(String input) { + if (input == null || input.isBlank()) + return ""; + + String[] parts = input.split("(?<=[a-z])(?=[A-Z])|[_\\s-]+"); + StringBuilder sb = new StringBuilder(); + + for (String part : parts) { + if (part.isEmpty()) + continue; + + String lower = part.toLowerCase(); + if (sb.length() == 0) { + sb.append(lower); + } else { + sb.append(Character.toUpperCase(lower.charAt(0))) + .append(lower.substring(1)); + } + } + return sb.toString(); + } + + public static boolean isNumericType(int sqlType, String typeName) { + return sqlType == Types.BIGINT + || sqlType == Types.DECIMAL + || sqlType == Types.DOUBLE + || sqlType == Types.FLOAT + || sqlType == Types.INTEGER + || sqlType == Types.NUMERIC + || sqlType == Types.REAL + || sqlType == Types.SMALLINT + || sqlType == Types.TINYINT + || (sqlType == Types.OTHER && typeName != null && typeName.equalsIgnoreCase("DECFLOAT")); + } + + public static boolean isStringType(int sqlType) { + return sqlType == Types.CHAR + || sqlType == Types.CLOB + || sqlType == Types.DATALINK + || sqlType == Types.LONGNVARCHAR + || sqlType == Types.LONGVARCHAR + || sqlType == Types.NCHAR + || sqlType == Types.NCLOB + || sqlType == Types.NVARCHAR + || sqlType == Types.VARCHAR; + } + + public static boolean isBinaryType(int sqlType) { + return sqlType == Types.BINARY + || sqlType == Types.BIT + || sqlType == Types.BLOB + || sqlType == Types.LONGVARBINARY + || sqlType == Types.VARBINARY; + } + + public static boolean isDateTimeType(int sqlType) { + return sqlType == Types.DATE + || sqlType == Types.TIME + || sqlType == Types.TIMESTAMP; + } + + public static String sanitizeXmlTag(String tag) { + if (tag == null || tag.isEmpty()) { + return "_column"; + } + + char firstChar = tag.charAt(0); + if (!Character.isLetter(firstChar) && firstChar != '_') { + tag = "_" + tag; + } + + return tag.replaceAll("[^a-zA-Z0-9_.-]", "_"); + } +} \ No newline at end of file diff --git a/src/main/java/com/minicircuits/formatter/CsvFormatter.java b/src/main/java/com/minicircuits/formatter/CsvFormatter.java new file mode 100644 index 0000000..17b91d0 --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/CsvFormatter.java @@ -0,0 +1,105 @@ +package com.minicircuits.formatter; + +import com.minicircuits.QueryResult; +import com.minicircuits.Utils; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.Properties; +import java.util.logging.Logger; +import javax.servlet.http.HttpServletResponse; + +public class CsvFormatter implements ResponseFormatter { + private static final Logger logger = Logger.getLogger(CsvFormatter.class.getName()); + private final Properties configProps; + private static final char DEFAULT_DELIMITER = ','; + private static final char DEFAULT_QUOTE = '"'; + + public CsvFormatter(Properties configProps) { + this.configProps = configProps; + } + + @Override + public void format(HttpServletResponse response, QueryResult result) throws IOException, SQLException { + response.setContentType("text/csv"); + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Disposition", "attachment; filename=data.csv"); + response.setBufferSize(8192); + + char delimiter = getDelimiter(); + + try (BufferedWriter writer = new BufferedWriter( + new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8), 8192)) { + + // Write header row + writeRow(writer, result.headers, delimiter); + + // Cache column types to avoid repeated metadata calls + int colCount = result.metadata.getColumnCount(); + int[] columnTypes = new int[colCount]; + String[] columnTypeNames = new String[colCount]; + for (int i = 0; i < colCount; i++) { + columnTypes[i] = result.metadata.getColumnType(i + 1); + columnTypeNames[i] = result.metadata.getColumnTypeName(i + 1); + } + + // Write data rows + int rowCount = 0; + while (result.resultSet.next()) { + String[] rowData = new String[colCount]; + for (int i = 1; i <= colCount; i++) { + String value = result.resultSet.getString(i); + if (value == null) { + rowData[i-1] = ""; + } else if (Utils.isNumericType(columnTypes[i-1], columnTypeNames[i-1])) { + rowData[i-1] = value.trim(); + } else { + rowData[i-1] = Utils.rightTrim(value); + } + } + writeRow(writer, rowData, delimiter); + rowCount++; + + // Flush periodically + if (rowCount % 1000 == 0) { + writer.flush(); + } + } + } + } + + private char getDelimiter() { + String delimiterStr = configProps.getProperty("csv.delimiter", ","); + return delimiterStr.isEmpty() ? DEFAULT_DELIMITER : delimiterStr.charAt(0); + } + + private void writeRow(BufferedWriter writer, String[] values, char delimiter) throws IOException { + for (int i = 0; i < values.length; i++) { + if (i > 0) { + writer.write(delimiter); + } + writer.write(escapeField(values[i], delimiter)); + } + writer.write("\r\n"); + } + + private String escapeField(String field, char delimiter) { + if (field == null) { + return ""; + } + + // If the field contains delimiter, quotes, or newlines, it needs to be quoted + if (field.indexOf(delimiter) >= 0 || field.indexOf(DEFAULT_QUOTE) >= 0 || + field.indexOf('\n') >= 0 || field.indexOf('\r') >= 0) { + + // Double up any quotes in the field + field = field.replace(String.valueOf(DEFAULT_QUOTE), + String.valueOf(DEFAULT_QUOTE) + String.valueOf(DEFAULT_QUOTE)); + + // Wrap the field in quotes + return DEFAULT_QUOTE + field + DEFAULT_QUOTE; + } + + return field; + } +} \ No newline at end of file diff --git a/src/main/java/com/minicircuits/formatter/ErrorFormatter.java b/src/main/java/com/minicircuits/formatter/ErrorFormatter.java new file mode 100644 index 0000000..b8603f1 --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/ErrorFormatter.java @@ -0,0 +1,71 @@ +package com.minicircuits.formatter; + +import com.minicircuits.Utils; +import java.io.IOException; +import java.io.PrintWriter; +import javax.servlet.http.HttpServletResponse; +import org.json.JSONObject; + +public class ErrorFormatter { + + public void format(HttpServletResponse response, int status, String message, String outputFormat) throws IOException { + response.setStatus(status); + + switch (outputFormat) { + case "xml": + formatXml(response, status, message); + break; + case "html": + formatHtml(response, status, message); + break; + case "csv": + formatCsv(response, status, message); + break; + default: + formatJson(response, status, message); + break; + } + } + + private void formatJson(HttpServletResponse response, int status, String message) throws IOException { + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + try (PrintWriter out = response.getWriter()) { + JSONObject obj = new JSONObject(); + obj.put("success", false); + obj.put("status", status); + obj.put("error", message != null ? message : "Unknown error"); + out.write(obj.toString()); + } + } + + private void formatXml(HttpServletResponse response, int status, String message) throws IOException { + response.setContentType("application/xml"); + response.setCharacterEncoding("UTF-8"); + try (PrintWriter out = response.getWriter()) { + out.println(""); + out.println(""); + out.println(" false"); + out.println(" " + status + ""); + out.println(" " + Utils.escapeXml(message) + ""); + out.println(""); + } + } + + private void formatHtml(HttpServletResponse response, int status, String message) throws IOException { + response.setContentType("text/html"); + response.setCharacterEncoding("UTF-8"); + try (PrintWriter out = response.getWriter()) { + out.println("

Error " + status + ": " + Utils.escapeHtml(message) + "

"); + } + } + + private void formatCsv(HttpServletResponse response, int status, String message) throws IOException { + response.setContentType("text/csv"); + response.setCharacterEncoding("UTF-8"); + try (PrintWriter out = response.getWriter()) { + out.println("success,status,error"); + out.println("false," + status + ",\"" + (message != null ? message.replace("\"", "\"\"") : "") + "\""); + } + } +} diff --git a/src/main/java/com/minicircuits/formatter/ExcelFormatter.java b/src/main/java/com/minicircuits/formatter/ExcelFormatter.java new file mode 100644 index 0000000..2c16022 --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/ExcelFormatter.java @@ -0,0 +1,135 @@ +package com.minicircuits.formatter; + +import com.minicircuits.QueryResult; +import com.minicircuits.StyleManager; +import java.io.*; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.Properties; +import java.util.logging.Logger; +import javax.servlet.http.HttpServletResponse; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.xssf.streaming.SXSSFSheet; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; + +public class ExcelFormatter implements ResponseFormatter { + private static final Logger logger = Logger.getLogger(ExcelFormatter.class.getName()); + private final Properties configProps; + private StyleManager styleManager; + + public ExcelFormatter(Properties configProps) { + this.configProps = configProps; + this.styleManager = new StyleManager(configProps); + } + + @Override + public void format(HttpServletResponse response, QueryResult result) throws IOException, SQLException { + logger.info("Starting Excel generation..."); + + // Don't set content type and headers until we're ready to write + response.reset(); + response.setBufferSize(32768); // Increase buffer size + + try (SXSSFWorkbook workbook = generateExcelWorkbook(result)) { + // Now that we have all data, set response headers + response.setContentType( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ); + response.setHeader( + "Content-Disposition", + "attachment; filename=ExcelExport.xlsx" + ); + + // Write to response with larger buffer + try ( + BufferedOutputStream out = new BufferedOutputStream( + response.getOutputStream(), + 32768 + ) + ) { + workbook.write(out); + out.flush(); + response.flushBuffer(); + } + + // Clear the style cache and dispose workbook + styleManager.clearCache(); + workbook.dispose(); + logger.info("Excel file generated successfully"); + + } catch (Exception e) { + logger.severe("Error generating Excel: " + e.getMessage()); + // If error occurs before headers are sent, we can still send error response + if (!response.isCommitted()) { + response.reset(); + response.setContentType("text/plain"); + response + .getWriter() + .println("Error generating Excel: " + e.getMessage()); + } + throw e; // Rethrow to be handled by caller + } + } + + private SXSSFWorkbook generateExcelWorkbook(QueryResult result) throws SQLException, IOException { + SXSSFWorkbook workbook = new SXSSFWorkbook(100); + workbook.setCompressTempFiles(true); + Sheet sheet = workbook.createSheet("Data"); + + CellStyle headerStyle = styleManager.createHeaderStyle(workbook); + createHeaderRow(sheet, result.headers, headerStyle); + + styleManager.setColumnStyles(sheet, result.metadata); + styleManager.setOptimalColumnWidths(sheet, result.headers, result.metadata); + + int rowNum = 1; + int batchSize = Integer.parseInt(configProps.getProperty("excel.batch.size", "1000")); + int currentBatch = 0; + + while (result.resultSet.next()) { + Row row = sheet.createRow(rowNum++); + populateDataRow(row, result.resultSet, result.metadata); + + currentBatch++; + if (currentBatch >= batchSize) { + ((SXSSFSheet) sheet).flushRows(batchSize); + currentBatch = 0; + } + } + + return workbook; + } + + private void createHeaderRow(Sheet sheet, String[] headers, CellStyle headerStyle) { + Row headerRow = sheet.createRow(0); + for (int i = 0; i < headers.length; i++) { + Cell cell = headerRow.createCell(i); + cell.setCellValue(headers[i]); + cell.setCellStyle(headerStyle); + } + } + + private void populateDataRow(Row row, ResultSet rs, ResultSetMetaData metadata) throws SQLException { + for (int i = 1; i <= metadata.getColumnCount(); i++) { + Cell cell = row.createCell(i-1); + Object value = rs.getObject(i); + + if (value == null) { + continue; + } + + if (value instanceof Number) { + cell.setCellValue(((Number) value).doubleValue()); + } else if (value instanceof java.sql.Date) { + cell.setCellValue((java.sql.Date) value); + } else if (value instanceof java.sql.Timestamp) { + cell.setCellValue((java.sql.Timestamp) value); + } else if (value instanceof Boolean) { + cell.setCellValue((Boolean) value); + } else { + cell.setCellValue(value instanceof String ? ((String)value).trim() : value.toString()); + } + } + } +} \ No newline at end of file diff --git a/src/main/java/com/minicircuits/formatter/FormatterFactory.java b/src/main/java/com/minicircuits/formatter/FormatterFactory.java new file mode 100644 index 0000000..b0a3ed6 --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/FormatterFactory.java @@ -0,0 +1,31 @@ +package com.minicircuits.formatter; + +import java.util.Properties; + +public class FormatterFactory { + + public static ResponseFormatter getFormatter(String format, Properties configProps) { + if ("excel".equalsIgnoreCase(format)) { + return new ExcelFormatter(configProps); + } else if ("html".equalsIgnoreCase(format)) { + return new HtmlFormatter(configProps); + } else if ("csv".equalsIgnoreCase(format)) { + return new CsvFormatter(configProps); + } else if ("xml".equalsIgnoreCase(format)) { + return new XmlFormatter(configProps); + } else if ("rest".equalsIgnoreCase(format)) { + return new RestFormatter(configProps); + } else { + // Default to JSON + return new JsonFormatter(configProps); + } + } + + public static ErrorFormatter createErrorFormatter() { + return new ErrorFormatter(); + } + + public static SuccessFormatter createSuccessFormatter() { + return new SuccessFormatter(); + } +} diff --git a/src/main/java/com/minicircuits/formatter/HtmlFormatter.java b/src/main/java/com/minicircuits/formatter/HtmlFormatter.java new file mode 100644 index 0000000..2ad9b27 --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/HtmlFormatter.java @@ -0,0 +1,107 @@ +package com.minicircuits.formatter; + +import com.minicircuits.QueryResult; +import com.minicircuits.Utils; +import java.io.*; +import java.sql.SQLException; +import java.util.Properties; +import java.util.logging.Logger; +import javax.servlet.http.HttpServletResponse; + +public class HtmlFormatter implements ResponseFormatter { + private static final Logger logger = Logger.getLogger(HtmlFormatter.class.getName()); + private final Properties configProps; + + public HtmlFormatter(Properties configProps) { + this.configProps = configProps; + } + + @Override + public void format(HttpServletResponse response, QueryResult result) throws IOException, SQLException { + response.setContentType("text/html"); + response.setCharacterEncoding("UTF-8"); + + try (PrintWriter out = response.getWriter()) { + writeHtmlStyles(out); + out.println("
"); + out.println(""); + writeHtmlHeaders(out, result.headers); + writeHtmlBody(out, result); + out.println("
"); + out.println("
"); + } + } + + private void writeHtmlStyles(PrintWriter out) { + out.println(""); + } + + private void writeHtmlHeaders(PrintWriter out, String[] headers) { + out.println(""); + for (String header : headers) { + out.println("" + Utils.escapeHtml(header) + ""); + } + out.println(""); + } + + private int writeHtmlBody(PrintWriter out, QueryResult result) throws SQLException { + int batchSize = Integer.parseInt( + configProps.getProperty("html.batch.size", "1000") + ); + int currentBatch = 0; + int rowCount = 0; + int colCount = result.metadata.getColumnCount(); + + while (result.resultSet.next()) { + out.println(""); + for (int i = 1; i <= colCount; i++) { + String value = result.resultSet.getString(i); + int columnType = result.metadata.getColumnType(i); + String columnTypeName = result.metadata.getColumnTypeName(i); + + if (Utils.isNumericType(columnType, columnTypeName)) { + out.print(""); + } else { + out.print(""); + } + out.print(Utils.escapeHtml(value != null ? Utils.rightTrim(value) : "")); + out.println(""); + } + out.println(""); + rowCount++; + + if (++currentBatch >= batchSize) { + out.flush(); + currentBatch = 0; + } + } + return rowCount; + } +} \ No newline at end of file diff --git a/src/main/java/com/minicircuits/formatter/JsonFormatter.java b/src/main/java/com/minicircuits/formatter/JsonFormatter.java new file mode 100644 index 0000000..897a32c --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/JsonFormatter.java @@ -0,0 +1,88 @@ +package com.minicircuits.formatter; + +import com.minicircuits.QueryResult; +import com.minicircuits.Utils; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Properties; +import javax.servlet.http.HttpServletResponse; + +public class JsonFormatter implements ResponseFormatter { + + public JsonFormatter(Properties configProps) { + // configProps not needed by this formatter + } + + @Override + public void format(HttpServletResponse response, QueryResult result) throws IOException, SQLException { + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + response.setBufferSize(8192); + + try (BufferedOutputStream bos = new BufferedOutputStream(response.getOutputStream(), 8192); + OutputStreamWriter osw = new OutputStreamWriter(bos, StandardCharsets.UTF_8)) { + + int colCount = result.metadata.getColumnCount(); + osw.write("["); + boolean firstRow = true; + int rowCount = 0; + + while (result.resultSet.next()) { + if (!firstRow) { + osw.write(","); + } + firstRow = false; + + osw.write("{"); + for (int i = 1; i <= colCount; i++) { + if (i > 1) osw.write(","); + String columnName = result.metadata.getColumnName(i); + String value = result.resultSet.getString(i); + + int columnType = result.metadata.getColumnType(i); + String columnTypeName = result.metadata.getColumnTypeName(i); + + osw.write("\""); + osw.write(Utils.escapeJson(columnName)); + osw.write("\":"); + if (value == null) { + osw.write("null"); + } else if (Utils.isNumericType(columnType, columnTypeName)) { + try { + String trimmed = value.trim(); + if (columnType == Types.FLOAT + || columnType == Types.DOUBLE + || columnType == Types.REAL + || result.metadata.getScale(i) > 0) { + osw.write(String.valueOf(Double.parseDouble(trimmed))); + } else { + osw.write(String.valueOf(Long.parseLong(trimmed))); + } + } catch (NumberFormatException e) { + osw.write("\""); + osw.write(Utils.escapeJson(Utils.rightTrim(value))); + osw.write("\""); + } + } else if (columnType == Types.BOOLEAN) { + osw.write(String.valueOf(result.resultSet.getBoolean(i))); + } else { + osw.write("\""); + osw.write(Utils.escapeJson(Utils.rightTrim(value))); + osw.write("\""); + } + } + osw.write("}"); + rowCount++; + + if (rowCount % 1000 == 0) { + osw.flush(); + } + } + + osw.write("]"); + osw.flush(); + } + } +} diff --git a/src/main/java/com/minicircuits/formatter/ResponseFormatter.java b/src/main/java/com/minicircuits/formatter/ResponseFormatter.java new file mode 100644 index 0000000..30ecc6c --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/ResponseFormatter.java @@ -0,0 +1,10 @@ +package com.minicircuits.formatter; + +import com.minicircuits.QueryResult; +import java.io.IOException; +import java.sql.SQLException; +import javax.servlet.http.HttpServletResponse; + +public interface ResponseFormatter { + void format(HttpServletResponse response, QueryResult result) throws IOException, SQLException; +} \ No newline at end of file diff --git a/src/main/java/com/minicircuits/formatter/RestFormatter.java b/src/main/java/com/minicircuits/formatter/RestFormatter.java new file mode 100644 index 0000000..b2977f3 --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/RestFormatter.java @@ -0,0 +1,156 @@ +package com.minicircuits.formatter; + +import com.minicircuits.QueryResult; +import com.minicircuits.Utils; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import javax.servlet.http.HttpServletResponse; + +public class RestFormatter implements ResponseFormatter { + + public RestFormatter(Properties configProps) { + // configProps not needed by this formatter + } + + @Override + public void format(HttpServletResponse response, QueryResult result) throws IOException, SQLException { + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + response.setBufferSize(8192); + + // Phase 1: Write data array to a temp file + Path dataTempFile = Files.createTempFile("fetchapi_data_", ".tmp"); + int rowCount = 0; + + try { + try (BufferedOutputStream dataBos = new BufferedOutputStream(new FileOutputStream(dataTempFile.toFile()), 8192); + OutputStreamWriter dataWriter = new OutputStreamWriter(dataBos, StandardCharsets.UTF_8)) { + + int colCount = result.metadata.getColumnCount(); + String[] keys = resolveColumnKeys(result); + + dataWriter.write("["); + + while (result.resultSet.next()) { + if (rowCount > 0) { + dataWriter.write(","); + } + + dataWriter.write("{"); + for (int i = 1; i <= colCount; i++) { + if (i > 1) dataWriter.write(","); + + String value = result.resultSet.getString(i); + int columnType = result.metadata.getColumnType(i); + String columnTypeName = result.metadata.getColumnTypeName(i); + + dataWriter.write("\""); + dataWriter.write(Utils.escapeJson(keys[i - 1])); + dataWriter.write("\":"); + + if (value == null) { + dataWriter.write("null"); + } else if (Utils.isNumericType(columnType, columnTypeName)) { + try { + String trimmed = value.trim(); + if (columnType == Types.FLOAT + || columnType == Types.DOUBLE + || columnType == Types.REAL + || result.metadata.getScale(i) > 0) { + dataWriter.write(String.valueOf(Double.parseDouble(trimmed))); + } else { + dataWriter.write(String.valueOf(Long.parseLong(trimmed))); + } + } catch (NumberFormatException e) { + dataWriter.write("\""); + dataWriter.write(Utils.escapeJson(Utils.rightTrim(value))); + dataWriter.write("\""); + } + } else if (columnType == Types.BOOLEAN) { + dataWriter.write(String.valueOf(result.resultSet.getBoolean(i))); + } else { + dataWriter.write("\""); + dataWriter.write(Utils.escapeJson(Utils.rightTrim(value))); + dataWriter.write("\""); + } + } + dataWriter.write("}"); + rowCount++; + } + + dataWriter.write("]"); + } + + // Phase 2: Write final envelope with records BEFORE data + try (BufferedOutputStream bos = new BufferedOutputStream(response.getOutputStream(), 8192); + OutputStreamWriter osw = new OutputStreamWriter(bos, StandardCharsets.UTF_8)) { + + osw.write("{\"success\":true,\"currentTimestamp\":\""); + osw.write(new Timestamp(System.currentTimeMillis()).toString()); + osw.write("\",\"records\":"); + osw.write(Integer.toString(rowCount)); + osw.write(",\"data\":"); + osw.flush(); + + // Copy data temp file to response + try (InputStream dataIn = new BufferedInputStream(new FileInputStream(dataTempFile.toFile()), 8192)) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = dataIn.read(buffer)) != -1) { + bos.write(buffer, 0, bytesRead); + } + } + + // Close the envelope - write directly to bos since we bypassed osw for the copy + bos.write("}".getBytes(StandardCharsets.UTF_8)); + bos.flush(); + } + + } finally { + Files.deleteIfExists(dataTempFile); + } + } + + /** + * Resolves all column keys, camelCasing them and appending a numeric + * suffix to duplicates: customerName, customerName2, customerName3, etc. + */ + private String[] resolveColumnKeys(QueryResult result) throws SQLException { + int colCount = result.metadata.getColumnCount(); + String[] keys = new String[colCount]; + Map seen = new HashMap<>(); + + for (int i = 1; i <= colCount; i++) { + String columnName = result.metadata.getColumnName(i); + String columnLabel = result.metadata.getColumnLabel(i); + + String key = resolveColumnKey(columnName, columnLabel); + int count = seen.getOrDefault(key, 0) + 1; + seen.put(key, count); + + keys[i - 1] = count == 1 ? key : key + count; + } + return keys; + } + + /** + * If column label exists camelCase it, else column name + */ + private String resolveColumnKey(String columnName, String columnLabel) { + String source; + if (columnLabel != null && !columnLabel.trim().isEmpty()) { + source = columnLabel.trim(); + } else { + source = columnName.trim(); + } + return Utils.toCamelCase(source); + } +} diff --git a/src/main/java/com/minicircuits/formatter/SuccessFormatter.java b/src/main/java/com/minicircuits/formatter/SuccessFormatter.java new file mode 100644 index 0000000..becf4dd --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/SuccessFormatter.java @@ -0,0 +1,67 @@ +package com.minicircuits.formatter; + +import com.minicircuits.Utils; +import java.io.IOException; +import java.io.PrintWriter; +import javax.servlet.http.HttpServletResponse; +import org.json.JSONObject; + +public class SuccessFormatter { + + public void format(HttpServletResponse response, String message, String outputFormat) throws IOException { + switch (outputFormat) { + case "xml": + formatXml(response, message); + break; + case "html": + formatHtml(response, message); + break; + case "csv": + formatCsv(response, message); + break; + default: + formatJson(response, message); + break; + } + } + + private void formatJson(HttpServletResponse response, String message) throws IOException { + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + try (PrintWriter out = response.getWriter()) { + JSONObject obj = new JSONObject(); + obj.put("success", true); + obj.put("message", message != null ? message : "SQL executed successfully"); + out.write(obj.toString()); + } + } + + private void formatXml(HttpServletResponse response, String message) throws IOException { + response.setContentType("application/xml"); + response.setCharacterEncoding("UTF-8"); + try (PrintWriter out = response.getWriter()) { + out.println(""); + out.println(""); + out.println(" true"); + out.println(" " + Utils.escapeXml(message) + ""); + out.println(""); + } + } + + private void formatHtml(HttpServletResponse response, String message) throws IOException { + response.setContentType("text/html"); + response.setCharacterEncoding("UTF-8"); + try (PrintWriter out = response.getWriter()) { + out.println("

Success: " + Utils.escapeHtml(message) + "

"); + } + } + + private void formatCsv(HttpServletResponse response, String message) throws IOException { + response.setContentType("text/csv"); + response.setCharacterEncoding("UTF-8"); + try (PrintWriter out = response.getWriter()) { + out.println("success,message"); + out.println("true,\"" + (message != null ? message.replace("\"", "\"\"") : "") + "\""); + } + } +} diff --git a/src/main/java/com/minicircuits/formatter/XmlFormatter.java b/src/main/java/com/minicircuits/formatter/XmlFormatter.java new file mode 100644 index 0000000..239167c --- /dev/null +++ b/src/main/java/com/minicircuits/formatter/XmlFormatter.java @@ -0,0 +1,86 @@ +package com.minicircuits.formatter; + +import com.minicircuits.QueryResult; +import com.minicircuits.Utils; +import java.io.*; +import java.sql.SQLException; +import java.util.Properties; +import java.util.logging.Logger; +import javax.servlet.http.HttpServletResponse; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamWriter; + +public class XmlFormatter implements ResponseFormatter { + private static final Logger logger = Logger.getLogger(XmlFormatter.class.getName()); + + public XmlFormatter(Properties configProps) { + // configProps not needed by this formatter + } + + @Override + public void format(HttpServletResponse response, QueryResult result) throws IOException, SQLException { + response.setContentType("application/xml"); + response.setCharacterEncoding("UTF-8"); + response.setBufferSize(8192); + + try { + // Create XML stream writer + XMLOutputFactory factory = XMLOutputFactory.newInstance(); + XMLStreamWriter writer = factory.createXMLStreamWriter( + response.getOutputStream(), "UTF-8"); + + // Start document + writer.writeStartDocument("UTF-8", "1.0"); + writer.writeStartElement("data"); + + // Write data rows + int rowCount = 0; + while (result.resultSet.next()) { + writer.writeStartElement("row"); + + for (int i = 1; i <= result.metadata.getColumnCount(); i++) { + String columnName = Utils.sanitizeXmlTag(result.metadata.getColumnName(i)); + String value = result.resultSet.getString(i); + + int columnType = result.metadata.getColumnType(i); + String columnTypeName = result.metadata.getColumnTypeName(i); + + writer.writeStartElement(columnName); + if (Utils.isNumericType(columnType, columnTypeName)) { + writer.writeAttribute("type", "numeric"); + } else if (Utils.isDateTimeType(columnType)) { + writer.writeAttribute("type", "datetime"); + } else if (Utils.isBinaryType(columnType)) { + writer.writeAttribute("type", "binary"); + } else if (columnType == java.sql.Types.BOOLEAN) { + writer.writeAttribute("type", "boolean"); + } else { + writer.writeAttribute("type", "string"); + } + if (value != null) { + writer.writeCharacters(Utils.rightTrim(value)); + } + writer.writeEndElement(); // column + } + + writer.writeEndElement(); // row + + // Flush periodically + if (++rowCount % 1000 == 0) { + writer.flush(); + } + } + + // End document + writer.writeEndElement(); // data + writer.writeEndDocument(); + writer.flush(); + writer.close(); + + } catch (XMLStreamException e) { + logger.severe("Error generating XML: " + e.getMessage()); + throw new IOException("Error generating XML", e); + } + } +} diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 0000000..fad1f72 --- /dev/null +++ b/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,31 @@ + + + EncodingFilter + com.minicircuits.CharacterEncodingFilter + + encoding + UTF-8 + + + forceEncoding + true + + + + EncodingFilter + /* + + + FetchAPIServlet + com.minicircuits.FetchAPI + 1 + + + FetchAPIServlet + /FetchAPIServlet + +