Detailed explanation of the implementation of Java character encoding and decoding

package com.ronsoft.books.nio.charset;

import java.nio.charset.Charset;
import java.nio.ByteBuffer;

/**
* Charset encoding test. Run the same input string, which contains some
* non-ascii characters, through several Charset encoders and dump out the hex
* values ​​of the resulting byte sequences.
*
* @author Ron Hitchens ([email protected])
*/
public class EncodeTest {
public static void main(String[] argv) throws Exception {
// This is the character sequence to encode
String input = "/u00bfMa/u00f1ana?";
// the list of charsets to encode with
String[] charsetNames = { "US-ASCII", "ISO-8859-1", "UTF-8",
"UTF-16BE", "UTF-16LE", "UTF-16" // , "X-ROT13"
};
for (int i = 0; i < charsetNames.length; i++) {
doEncode(Charset.forName(charsetNames[i]), input);
}
}

/**
* For a given Charset and input string, encode the chars and print out the
* resulting byte encoding in a readable form.
*/
private static void doEncode(Charset cs, String input) {
ByteBuffer bb = cs.encode(input);
System.out.println("Charset: " + cs.name());
System.out.println(" Input: " + input);
System.out.println("Encoded: ");
for (int i = 0; bb.hasRemaining(); i++) {
int b = bb.get();
int ival = ((int) b) & 0xff;
char c = (char) ival;
// Keep tabular alignment pretty
if (i < 10)
System.out.print(" ");
//Print index number
System.out.print(" " + i + ": ");
// Better formatted output is coming someday...
if (ival < 16)
System.out.print("0");
// Print the hex value of the byte
System.out.print(Integer.toHexString(ival));
// If the byte seems to be the value of a
// printable character, print it. No guarantee
// it will be.
if (Character.isWhitespace(c) || ​​Character.isISOControl(c)) {
System.out.println("");
} else {
System.out.println(" (" + c + ")");
}
}
System.out.println("");
}
}

Charset: ISO-8859-1
Input: ?Ma?ana?
Encoded:
0:20
1: bf (?)
2: 4d (M)
3:61(a)
4: f1 (?)
5:61(a)
6: 6e(n)
7:61(a)
8: 3f (?)

Charset: UTF-8
Input: ?Ma?ana?
Encoded:
0:20
1: c2 (?)
2: bf (?)
3: 4d (M)
4:61(a)
5: c3 (?)
6: b1 (±)
7:61(a)
8: 6e(n)
9:61(a)
10: 3f (?)

Charset: UTF-16BE
Input: ?Ma?ana?
Encoded:
0:00
1:20
2:00
3: bf (?)
4:00
5: 4d (M)
6:00
7:61(a)
8:00
9: f1 (?)
10:00
11:61(a)
12:00
13: 6e(n)
14:00
15: 61 (a)
16:00
17: 3f (?)

Charset: UTF-16LE
Input: ?Ma?ana?
Encoded:
0:20
1:00
2: bf (?)
3:00
4: 4d (M)
5:00
6:61(a)
7:00
8: f1 (?)
9:00
10:61(a)
11:00
12: 6e(n)
13:00
14: 61 (a)
15:00
16: 3f (?)
17:00

Charset: UTF-16
Input: ?Ma?ana?
Encoded:
0: fe (?)
1: ff (?)
2:00
3:20
4:00
5: bf (?)
6:00
7: 4d (M)
8:00
9:61(a)
10:00
11: f1 (?)
12:00
13: 61 (a)
14:00
15: 6e(n)
16:00
17: 61 (a)
18:00
19: 3f (?)

package java.nio.charset;
public abstract class Charset implements Comparable
{
public static boolean isSupported (String charsetName)
public static Charset forName (String charsetName)
public static SortedMap availableCharsets()
public final String name()
public final Set aliases()
public String displayName()
public String displayName (Locale locale)
public final boolean isRegistered()
public boolean canEncode()
public abstract CharsetEncoder newEncoder();
public final ByteBuffer encode (CharBuffer cb)
public final ByteBuffer encode (String str)
public abstract CharsetDecoder newDecoder();
public final CharBuffer decode (ByteBuffer bb)
public abstract boolean contains (Charset cs);
public final boolean equals (Object ob)
public final int compareTo (Object ob)
public final int hashCode()
public final String toString()
}

public abstract class Charset implements Comparable
{
// This is a partial API listing
public abstract boolean contains (Charset cs);
public final boolean equals (Object ob)
public final int compareTo (Object ob)
public final int hashCode()
public final String toString()
}

package java.nio.charset;
public abstract class CharsetEncoder
{
// This is a partial API listing
public boolean canEncode (char c)
public boolean canEncode (CharSequence cs)
}

package java.nio.charset;
public class CoderResult {
public static final CoderResult OVERFLOW
public static final CoderResult UNDERFLOW
public boolean isUnderflow()
public boolean isOverflow()
<span style="white-space:pre"> </span>public boolean isError()
public boolean isMalformed()
public boolean isUnmappable()
public int length()
public static CoderResult malformedForLength (int length)
public static CoderResult unmappableForLength (int length)
<span style="white-space:pre"> </span>public void throwException() throws CharacterCodingException
}

package java.nio.charset;
public abstract class CharsetDecoder
{
// This is a partial API listing
public final CharsetDecoder reset()
public final CharBuffer decode (ByteBuffer in)
throws CharacterCodingException
public final CoderResult decode (ByteBuffer in, CharBuffer out,
boolean endOfInput)
public final CoderResult flush (CharBuffer out)
}

package com.ronsoft.books.nio.charset;

import java.nio.*;
import java.nio.charset.*;
import java.nio.channels.*;
import java.io.*;

/**
* Test charset decoding.
*
* @author Ron Hitchens ([email protected])
*/
public class CharsetDecode {
/**
* Test charset decoding in the general case, detecting and handling buffer
* under/overflow and flushing the decoder state at end of input. This code
* reads from stdin and decodes the ASCII-encoded byte stream to chars. The
* decoded chars are written to stdout. This is effectively a 'cat' for
* input ascii files, but another charset encoding could be used by simply
* specifying it on the command line.
*/
public static void main(String[] argv) throws IOException {
// Default charset is standard ASCII
String charsetName = "ISO-8859-1";
// Charset name can be specified on the command line
if (argv. length > 0) {
charsetName = argv[0];
}
// Wrap a Channel around stdin, wrap a channel around stdout,
// find the named Charset and pass them to the deco de method.
// If the named charset is not valid, an exception of type
// UnsupportedCharsetException will be thrown.
decodeChannel(Channels.newChannel(System.in), new OutputStreamWriter(
System.out), Charset.forName(charsetName));
}

/**
* General purpose static method which reads bytes from a Channel, decodes
* them according
*
* @param source
* A ReadableByteChannel object which will be read to EOF as a
* source of encoded bytes.
* @param writer
* A Writer object to which decoded chars will be written.
* @param charset
* A Charset object, whose CharsetDecoder will be used to do the
* character set decoding. Java NIO 206
*/
public static void decodeChannel(ReadableByteChannel source, Writer writer,
Charset charset) throws UnsupportedCharsetException, IOException {
// Get a decoder instance from the Charset
CharsetDecoder decoder = charset.newDecoder();
// Tell decoder to replace bad chars with default mark
decoder.onMalformedInput(CodingErrorAction.REPLACE);
decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
// Allocate radically different input and output buffer sizes
// for testing purposes
ByteBuffer bb = ByteBuffer.allocateDirect(16 * 1024);
CharBuffer cb = CharBuffer.allocate(57);
// Buffer starts empty; indicate input is needed
CoderResult result = CoderResult.UNDERFLOW;
boolean eof = false;
while (!eof) {
// Input buffer underflow; decoder wants more input
if (result == CoderResult.UNDERFLOW) {
// decoder consumes all input, prepare to refill
bb.clear();
// Fill the input buffer; watch for EOF
eof = (source.read(bb) == -1);
// Prepare the buffer for reading by decoder
bb.flip();
}
// Decode input bytes to output chars; pass EOF flag
result = decoder.decode(bb, cb, eof);
// If output buffer is full, drain output
if (result == CoderResult.OVERFLOW) {
drainCharBuf(cb, writer);
}
}
// Flush any remaining state from the decoder, being careful
// to detect output buffer overflow(s)
while (decoder.flush(cb) == CoderResult.OVERFLOW) {
drainCharBuf(cb, writer);
}
// Drain any chars remaining in the output buffer
drainCharBuf(cb, writer);
// Close the channel; push out any buffered data to stdout
source.close();
writer.flush();
}

/**
* Helper method to drain the char buffer and write its content to the given
* Writer object. Upon return, the buffer is empty and ready to be refilled.
*
* @param cb
* A CharBuffer containing chars to be written.
* @param writer
* A Writer object to consume the chars in cb.
*/
static void drainCharBuf(CharBuffer cb, Writer writer) throws IOException {
cb.flip(); // Prepare buffer for draining
// This writes the chars contained in the CharBuffer but
// doesn't actually modify the state of the buffer.
// If the char buffer was being drained by calls to get( ),
// a loop might be needed here.
if (cb.hasRemaining()) {
writer.write(cb.toString());
}
cb.clear(); // Prepare buffer to be filled again
}
}

package com.ronsoft.books.nio.charset;

import java.nio.CharBuffer;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
import java.util.Map;
import java.util.Iterator;
import java.io.Writer;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.OutputStreamWriter;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.FileReader;

/**
* A Charset implementation which performs Rot13 encoding. Rot -13 encoding is a
* simple text obfuscation algorithm which shifts alphabetical characters by 13
* so that 'a' becomes 'n', 'o' becomes 'b', etc. This algorithm was popularized
* by the Usenet discussion forums many years ago to mask naughty words, hide
* answers to questions, and so on. The Rot13 algorithm is symmetrical, applying
* it to text that has been scrambled by Rot13 will give you the original
* unscrambled text.
*
* Applying this Charset encoding to an output stream will cause everything you
* write to that stream to be Rot13 scrambled as it's written out. And appying
* it to an input stream causes data read to be Rot13 descrambled as it's read.
*
* @author Ron Hitchens ([email protected])
*/
public class Rot13Charset extends Charset {
// the name of the base charset encoding we delegate to
private static final String BASE_CHARSET_NAME = "UTF-8";
// Handle to the real charset we'll use for transcoding between
// characters and bytes. Doing this allows us to apply the Rot13
// algorithm to multibyte charset encodings. But only the
// ASCII alpha chars will be rotated, regardless of the base encoding.
Charset baseCharset;

/**
* Constructor for the Rot13 charset. Call the superclass constructor to
* pass along the name(s) we'll be known by. Then save a reference to the
* delegate Charset.
*/
protected Rot13Charset(String canonical, String[] aliases) {
super(canonical, aliases);
// Save the base charset we're delegating to
baseCharset = Charset.forName(BASE_CHARSET_NAME);
}

//------------------------------------------------ ----------
/**
* Called by users of this Charset to obtain an encoder. This implementation
* instantiates an instance of a private class (defined below) and passes it
* an encoder from the base Charset.
*/
public CharsetEncoder newEncoder() {
return new Rot13Encoder(this, baseCharset.newEncoder());
}

/**
* Called by users of this Charset to obtain a decoder. This implementation
* instantiates an instance of a private class (defined below) and passes it
* a decoder from the base Charset.
*/
public CharsetDecoder newDecoder() {
return new Rot13Decoder(this, baseCharset.newDecoder());
}

/**
* This method must be implemented by concrete Charsets. We always say no,
* which is safe.
*/
public boolean contains(Charset cs) {
return (false);
}

/**
* Common routine to rotate all the ASCII alpha chars in the given
* CharBuffer by 13. Note that this code explicitly compares for upper and
* lower case ASCII chars rather than using the methods
* Character.isLowerCase and Character.isUpperCase. This is because the
* rotate-by-13 scheme only works properly for the alphabetic characters of
* the ASCII charset and those methods can return true for non-ASCII Unicode
* chars.
*/
private void rot13(CharBuffer cb) {
for (int pos = cb.position(); pos < cb.limit(); pos++) {
char c = cb.get(pos);
char a = '/u0000';
// Is it lowercase alpha?
if ((c >= 'a') && (c <= 'z')) {
a = 'a';
}
// Is it uppercase alpha?
if ((c >= 'A') && (c <= 'Z')) {
a = 'A';
}
// If either, roll it by 13
if (a != '/u0000') {
c = (char) ((((c - a) + 13) % 26) + a);
cb.put(pos, c);
}
}
}

//------------------------------------------------ --------
/**
* The encoder implementation for the Rot13 Chars et. This class, and the
* matching decoder class below, should also override the "impl" methods,
* such as implOnMalformedInput( ) and make passthrough calls to the
* baseEncoder object. That is left as an exercise for the hacker.
*/
private class Rot13Encoder extends CharsetEncoder {
private CharsetEncoder baseEncoder;

/**
* Constructor, call the superclass constructor with the Charset object
* and the encodings sizes from the delegate encoder.
*/
Rot13Encoder(Charset cs, CharsetEncoder baseEncoder) {
super(cs, baseEncoder.averageBytesPerChar(), baseEncoder
.maxBytesPerChar());
this.baseEncoder = baseEncoder;
}

/**
* Implementation of the encoding loop. First, we apply the Rot13
* scrambling algorithm to the CharBuffer, then reset the encoder for
* the base Charset and call it's encode( ) method to do the actual
* encoding. This may not work properly for non-Latin charsets. The
* CharBuffer passed in may be read -only or re-used by the caller for
* other purposes so we duplicate it and apply the Rot13 encoding to the
* copy. We DO want to advance the position of the input buffer to
* reflect the chars consumed.
*/
protected CoderResult encodeLoop(CharBuffer cb, ByteBuffer bb) {
CharBuffer tmpcb = CharBuffer.allocate(cb.remaining());
while (cb.hasRemaining()) {
tmpcb.put(cb.get());
}
tmpcb.rewind();
rot13(tmpcb);
baseEncoder.reset();
CoderResult cr = baseEncoder.encode(tmpcb, bb, true);
// If error or output overflow, we need to adjust
// the position of the input buffer to match what
// was really consumed from the temp buffer. If
// underflow (all input consumed), this is a no-op.
cb.position(cb.position() - tmpcb.remaining());
return(cr);
}
}

//------------------------------------------------ --------
/**
* The decoder implementation for the Rot13 Charset.
*/
private class Rot13Decoder extends CharsetDecoder {
private CharsetDecoder baseDecoder;

/**
* Constructor, call the superclass constructor with the Charset object
* and pass alon the chars/byte values ​​from the delegate decoder.
*/
Rot13Decoder(Charset cs, CharsetDecoder baseDecoder) {
super(cs, baseDecoder.averageCharsPerByte(), baseDecoder
.maxCharsPerByte());
this.baseDecoder = baseDecoder;
}

/**
* Implementation of the decoding loop. First, we reset the decoder for
* the base charset, then call it to decode the bytes into characters,
* saving the result code. The CharBuffer is then de-scrambled with the
* Rot13 algorithm and the result code is returned. This may not work
* properly for non-Latin charsets.
*/
protected CoderResult decodeLoop(ByteBuffer bb, CharBuffer cb) {
baseDecoder.reset();
CoderResult result = baseDecoder.decode(bb, cb, true);
rot13(cb);
return (result);
}
}

//------------------------------------------------ --------
/**
* Unit test for the Rot13 Charset. This main( ) will open and read an input
* file if named on the command line, or stdin if no args are provided, and
* write the contents to stdout via the X -ROT13 charset encoding. The
* "encryption" implemented by the Rot13 algorithm is symmetrical. Feeding
* in a plain-text file, such as Java source code for example, will output a
* scrambled version. Feeding the scrambled version back in will yield the
* original plain-text document.
*/
public static void main(String[] argv) throws Exception {
BufferedReader in;
if (argv. length > 0) {
// Open the named file
in = new bufferedReader (New FILEREADER (ARGV [0]));
} else {
// wrap a bufferedReader around stdin
in = new bufferedReader (New InputStreamReader (System.in));
}
// Create a Printstream that uses the rot13 encoding
Printstream out = New Printstream (System.out, FALSE, "X -Rot13");
String s = null;
// Read all input and write it to the output.
// as the data passes through the printstream,
// It will be rot13-enCoded.
While ((s = in.readline ())! = NULL) {
out.println (s);
}
out.flush();
}
}

package com.ronsoft.books.nio.charset;

Import java.nio.charset.charset;
Import java.nio.charset.spi.charsetProvider;
import java.util.HashSet;
Import java.util.iterator;

/**
* A CharsetProvider Class Which Makes Available The Charsets PROVIDD BY
* Ronsoft. Currently there is only one, namely the x -rot13 charset. This is
* Not a registered ichuset, so it's name before "x-" to avoid name
* Clastes with Office Charsets.
*
* To activate this charsetProvider, it's necessary to add a file to the
* ClassPath of the JVM Runtime at the Following Location:
* Meta-INF/Services/Java.nio.charsets.spi.charsetp Rovider
*
* That File Must Contain a Line with the Fully Qualify name of this class on
* a line by itset: com.ronsoft.books.nio.charsOftCharsetProvider Java
* Nio 216
*
* SEE The Javadoc Page for Java.nio.Charsets.spi.CharsetProvider for Full
* Details.
*
* @Author Ron Hitchens ([email protected])
*/
public class ronsoftcharsetProvider
// The name of the charset we provide
Private Static Final String Charset_name = "X-Rot13";
// a handle to the charset object
Private Charset Rot13 = NULL;

/**
* Constructionor, Instantiate a Charset object and save the reference.
*/
public ronsoftCharsetProvider () {
this.rot13 = new rot13Charset (charset_name, new string [0]);
}

/**
* Called by Charset Static Methods to Find A PARTICular Named Charset. If.
* It's the name of this charset (we do'T'T has been after any aliases) then return the
* Rot13 charset, else return null.
*/
Public Charset CharsetForname (String CharsetName) {{
if (charsetName.equalSignorecase (charset_name)) {{
Return (rot13);
}
Return (null);
}

/**
* Return an itrator over the set of charset objects we provide.
*
* @Return an itrator object containing references to all the charset
* Objects Provided by this class.
*/
public itrator <charset> charsets () {) {
Have set <cHarset> set = New HashSet <Charset> (1);
set.add (rot13);
Return (set.iterator ());
}
}

Add X -ROT13 to the character set list in Example 6-1 to generate this extra output:

Charset: X-Rot13
Input: anMaana?
Encoded:
0: C2 (()
1: BF (()
2: 5A (Z)
3: 6e (n)
4: C3 (()
5: B1 (±)
6: 6e (n)
7: 61 (A)
8: 6E (n)
9: 3F (?)